diff --git a/.gitignore b/.gitignore index b73bda620d62a0e9e4bc0c7ffd089a3a32bbed1f..4ade85f589b2911e7467bd5390a29a088dbd3f8d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,16 @@ -### C ### +### cmp_tool ### # Executables cmp_tool !test/cmp_tool +#cmp_tool files +*.cfg +*.info +*.dat +*.cmp + + +### C ### # Prerequisites *.d @@ -50,3 +58,19 @@ cmp_tool # C/C++/ObjC language server .ccls-cache + +### Gcov ### +# gcc coverage testing tool files +*.gcno +*.gcda +*.gcov +out + + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + + diff --git a/CHANGELOG.md b/CHANGELOG.md index f74a06c7db46a58d1de9780de5e0f7722cc58da0..1d940312d2e2a338b6228c2105b235b9beabdd37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [0.07] - 13-12-2021 +- **NOTE:** The behaviour of the cmp_tool has changed. From now on, the compressed data will be preceded by a header by default. The old behaviour can be achieved with the --no_header option the behaviour of the cmp_tool has changed. From now on, the compressed data will be preceded by a header by default. The old behaviour can be achieved with the --no_header option. +- Implement the compression entity header as defined in PLATO-UVIE-PL-UM-0001 for compression and decompression +- small bug fixes ## [0.06] - 12-11-2021 - if samples = 0 the cmp_tool counts the samples in the data file and uses them - if buffer_length = 0 the 2*samples parameter is used as buffer_length diff --git a/Makefile b/Makefile index 28c93b0de9d9d273e1bd1656a0996b0b389bbd14..f59c1ce4f0379592970e36588da2a4ae360ff208 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,10 @@ CC = gcc SOURCEDIR = lib INCLUDEDIR = include BUILDDIR = ./ -CFLAGS := -Wall -Wextra -std=gnu99 -pedantic -Wshadow \ +CFLAGS := -Wall -Wextra -std=gnu99 -mno-ms-bitfields -pedantic -Wshadow \ -Wunreachable-code #-Wdocumentation #-Wsign-conversion RELCFLAGS := -O2 # Release flags -DBCFLAGS := -O0 -g3 #debug flags +DBCFLAGS := -O0 -g3 -fsanitize=address -fsanitize=undefined #debug flags COVFLAGS := -fprofile-arcs -ftest-coverage #coverage flags CPPFLAGS := -I $(INCLUDEDIR) LDFLAGS := -lm @@ -21,7 +21,8 @@ SOURCES := cmp_tool.c \ $(SOURCEDIR)/decmp.c \ $(SOURCEDIR)/rdcu_pkt_to_file.c \ $(SOURCEDIR)/cmp_guess.c \ - $(SOURCEDIR)/cmp_tool_lib.c + $(SOURCEDIR)/cmp_tool_lib.c \ + $(SOURCEDIR)/cmp_entity.c TARGET := cmp_tool DEBUG?=1 diff --git a/README.md b/README.md index 66378e4532ab428e264fa94f5d54744bb6e66d51..2b2802ff04774a5e48eebeb156a19ee2fa127026 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,18 @@ Compiled executables can be found [here][3]. | Options | Description | |:------------------|:------------------------------------------------------------------------------| | `-h, --help` | Print some help text and exit | +| `-o <prefix>` | Use the `<prefix>` for output files<sup>[1](#fnote1)</sup> | +| `-n, --model_cfg` | Print a default model configuration and exit<sup>[2](#fnote2)</sup> | +| `--diff_cfg` | Print a default 1d-differencing configuration and exit<sup>[2](#fnote2)</sup> | +| `--no_header` | Do not add a compression entity header in front of the compressed data | +| `-a, --rdcu_par` | Add additional RDCU control parameters | | `-V, --version` | Print program version and exit | | `-v, --verbose` | Print various debugging information | -| `-n, --model_cfg` | Print a default model configuration and exit<sup>[1](#fnote1)</sup> | -| `--diff_cfg` | Print a default 1d-differencing configuration and exit<sup>[1](#fnote1)</sup> | -| `-a, --rdcu_par` | Add additional RDCU control parameters | -| `-o <prefix>` | Use the `<prefix>` for output files<sup>[2](#fnote2)</sup> | -<a name="fnote1">1</a>) **NOTE:** In the default configurations the **samples** -and **buffer_length** parameter is set to **0**! -<a name="fnote2">2</a>) **NOTE:** If the -o option is not used the `<prefix>` -will be set to "OUTPUT". +<a name="fnote2">1</a>) **NOTE:** If the -o option is not used the `<prefix>` +will be set to "OUTPUT". +<a name="fnote2">2</a>) **NOTE:** In the default configurations the **samples** +and **buffer_length** parameter is set to **0**! ### Compression Options @@ -41,11 +42,11 @@ The generated packets can be found in the `TC_FILES` directory. ### Decompression Options -| Options | Description | -|:----------- |:-------------------------------------------------| -| `-i <file>` | File containing the decompression information | -| `-d <file>` | File containing the compressed data | -| `-m <file>` | File containing the model of the compressed data | +| Options | Description | +|:----------- |:--------------------------------------------------------------------------------| +| `-d <file>` | File containing the compressed data | +| `-m <file>` | File containing the model of the compressed data | +| `-i <file>` | File containing the decompression information (required if --no_header was used)| ### Guessing Options @@ -81,6 +82,7 @@ For example: `12 AB 34 CD`. ## How to use the tool A simple example to show how the compression tool works. +Instructions on how to perform compression without headers can be found [here](how_to_no_header.md). 0. Download the [tool][3] or run `make` to build the tool @@ -91,39 +93,33 @@ A simple example to show how the compression tool works. `./cmp_tool --diff_cfg > cfg/default_config_1d.cfg` * To create a default model mode configuration: `./cmp_tool --model_cfg > cfg/default_config_model.cfg` -* Change the the **`samples`** and **`buffer_length`** parameters from `0` to `50` -in the `default_config_1d.cfg` and `default_config_model.cfg` files 2. Compress data with the default configurations: * Create a directory for the compressed data `mkdir compressed` * 1d-differencing mode compression `./cmp_tool -c cfg/default_config_1d.cfg -d test_data/test_data1.dat -o compressed/data1` - This creates these two files in the compressed directory: + This creates a files in the compressed directory: `data1.cmp` -> compressed `test_data1.dat` file - `data1.info` -> decompression information for `data1.cmp` * Model mode compression `./cmp_tool -c cfg/default_config_model.cfg -d test_data/test_data2.dat -m test_data/test_data1.dat -o compressed/data2` - We use `test_data1.dat` as the first model for `test_data2.dat`. - - Creates these three files in the compressed directory: + We use `test_data1.dat` as the first model for `test_data2.dat`. + Creates these two files in the compressed directory: `data2.cmp ` -> compressed `test_data3.dat` file - `data2.info` -> decompression information for `data2.cmp` `data2_upmodel.dat` -> updated model used to **compress** the next data in model mode 3. Decompress the data * Create a directory for the decompressed data `mkdir decompressed` * Decompress `data1.cmp` - `./cmp_tool -i compressed/data1.info -d compressed/data1.cmp -o decompressed/test_data1` - Creates this file in the decompressed directory: - `test_data1.dat ` -> decompressed `data1.cmp` file + `./cmp_tool -d compressed/data1.cmp -o decompressed/test_data1` + This creates a file in the decompressed directory: + `test_data1.dat` -> decompressed `data1.cmp` file * Decompress `data2.cmp` - `./cmp_tool -i compressed/data2.info -d compressed/data2.cmp -m decompressed/test_data1.dat -o decompressed/test_data2` + `./cmp_tool -d compressed/data2.cmp -m decompressed/test_data1.dat -o decompressed/test_data2` As for the compression we use `test_data1.dat` as our model for decompression. - Creates these two files in the decompressed directory: - `test_data2.dat ` -> decompressed `data2.cmp` file + `test_data2.dat` -> decompressed `data2.cmp` file `data2_upmodel.dat` -> updated model used to **decompress** the next data in model mode 4. Bonus: Check if the decompressed data are equal to the original data diff --git a/cmp_tool.c b/cmp_tool.c index a314b8f5b3b844718a4adbd4a67c1fa0a8d7cfbe..dddca2c7b1ae7380ccd291f1af48f83b86475758 100755 --- a/cmp_tool.c +++ b/cmp_tool.c @@ -23,19 +23,25 @@ #include <stdlib.h> #include <limits.h> #include <getopt.h> +#include <time.h> #include "include/cmp_tool_lib.h" #include "include/cmp_icu.h" #include "include/decmp.h" #include "include/cmp_guess.h" +#include "include/cmp_entity.h" #include "include/rdcu_pkt_to_file.h" -#define VERSION "0.06" +#define VERSION "0.07" #define BUFFER_LENGTH_DEF_FAKTOR 2 +#define DEFAULT_MODEL_ID 53264 /* random id used as default */ +#define DEFAULT_MODEL_COUNTER 0 + + /* * For long options that have no equivalent short option, use a * non-character as a pseudo short option, starting with CHAR_MAX + 1. @@ -46,7 +52,9 @@ enum { GUESS_LEVEL, RDCU_PKT_OPTION, LAST_INFO, - + NO_HEADER, + MODEL_ID, + MODEL_COUTER, }; static const struct option long_options[] = { @@ -60,6 +68,9 @@ static const struct option long_options[] = { {"guess", required_argument, NULL, GUESS_OPTION}, {"guess_level", required_argument, NULL, GUESS_LEVEL}, {"last_info", required_argument, NULL, LAST_INFO}, + {"no_header", no_argument, NULL, NO_HEADER}, + {"model_id", required_argument, NULL, MODEL_ID}, + {"model_counter", required_argument, NULL, MODEL_COUTER}, {NULL, 0, NULL, 0} }; @@ -80,6 +91,8 @@ static const char *last_info_file_name; /* if non zero print additional verbose output */ static int verbose_en; +/* if non zero add a compression entity header in front of the compressed data */ +static int include_cmp_header = 1; /* find a good set of compression parameters for a given dataset */ static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode, @@ -89,9 +102,15 @@ static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode, static int compression(struct cmp_cfg *cfg, struct cmp_info *info); /* decompress the data and write the results in file(s)*/ -static int decompression(uint32_t *decomp_input_buf, uint16_t *input_model_buf, +static int decompression(uint32_t *cmp_data_adr, uint16_t *input_model_buf, struct cmp_info *info); +/* model ID string set by the --model_id option */ +static const char *model_id_str; + +/* model counter string set by the --model_counter option */ +static const char *model_counter_str; + /** * @brief This is the main function of the compression / decompression tool @@ -124,9 +143,12 @@ int main(int argc, char **argv) struct cmp_cfg cfg = {0}; /* compressor configuration struct */ struct cmp_info info = {0}; /* decompression information struct */ - uint32_t *decomp_input_buf = NULL; + /* buffer containing all read in compressed data for decompression (including header if used) */ + void *decomp_input_buf = NULL; + /* address to the compressed data for the decompression */ + uint32_t *cmp_data_adr = NULL; + /* buffer containing the read in model */ uint16_t *input_model_buf = NULL; - ssize_t size; /* show help if no arguments are provided */ if (argc < 2) { @@ -153,6 +175,7 @@ int main(int argc, char **argv) break; case 'i': info_file_name = optarg; + include_cmp_header = 0; break; case 'm': /* read model */ model_file_name = optarg; @@ -180,12 +203,20 @@ int main(int argc, char **argv) case GUESS_LEVEL: guess_level = atoi(optarg); break; + case LAST_INFO: + last_info_file_name = optarg; + /* fall through */ case RDCU_PKT_OPTION: rdcu_pkt_mode = 1; + /* fall through */ + case NO_HEADER: + include_cmp_header = 0; break; - case LAST_INFO: - rdcu_pkt_mode = 1; - last_info_file_name = optarg; + case MODEL_ID: + model_id_str = optarg; + break; + case MODEL_COUTER: + model_counter_str = optarg; break; default: print_help(program_name); @@ -194,16 +225,18 @@ int main(int argc, char **argv) } } argc -= optind; - argv += optind; - if (argc > 0) { +#ifdef ARGUMENT_INPUT_MODE + + argv += optind; + if (argc > 2) { printf("%s: To many arguments.\n", PROGRAM_NAME); print_help(argv[0]); exit(EXIT_FAILURE); } -#if 0 + if (argc > 0) { - if(!data_file_name) + if (!data_file_name) data_file_name = argv[0]; else { printf("You can define the data file using either the -d option or the first argument, but not both.\n"); @@ -212,7 +245,7 @@ int main(int argc, char **argv) } } if (argc > 1) { - if(!model_file_name) + if (!model_file_name) model_file_name = argv[1]; else { printf("You can define the model file using either the -m option or the second argument, but not both.\n"); @@ -220,6 +253,12 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } } +#else + if (argc > 0) { + printf("%s: To many arguments.\n", PROGRAM_NAME); + print_help(argv[0]); + exit(EXIT_FAILURE); + } #endif if (print_model_cfg == 1) { @@ -235,7 +274,9 @@ int main(int argc, char **argv) printf("#########################################################\n"); printf("### PLATO Compression/Decompression Tool Version %s ###\n", VERSION); - printf("#########################################################\n\n"); + printf("#########################################################\n"); + if (!strcmp(VERSION, "0.07") || !strcmp(VERSION, "0.08")) + printf("Info: Note that the behaviour of the cmp_tool has changed. From now on, the compressed data will be preceded by a header by default. The old behaviour can be achieved with the --no_header option.\n\n"); if (!data_file_name) { fprintf(stderr, "%s: No data file (-d option) specified.\n", @@ -243,44 +284,38 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } - if (!cfg_file_name && !info_file_name && !guess_operation) { + if (!cfg_file_name && !info_file_name && !guess_operation && !include_cmp_header) { fprintf(stderr, "%s: No configuration file (-c option) or decompression information file (-i option) specified.\n", PROGRAM_NAME); exit(EXIT_FAILURE); } - /* read input data and .cfg or .info */ - if (cmp_operation || guess_operation) { /* compression mode */ + + if (cmp_operation || guess_operation) { + ssize_t size; + if (cmp_operation) { - /* printf("### Compression ###\n"); */ + printf("## Starting the compression ##\n"); printf("Importing configuration file %s ... ", cfg_file_name); error = read_cmp_cfg(cfg_file_name, &cfg, verbose_en); if (error) goto fail; - else - printf("DONE\n"); - + printf("DONE\n"); + } else { + printf("## Search for a good set of compression parameters ##\n"); } printf("Importing data file %s ... ", data_file_name); - /* count the samples in the data file when samples == 0 */ if (cfg.samples == 0) { size = read_file16(data_file_name, NULL, 0, 0); - if (size < 0) + if (size <= 0) /* empty file is treated as an error */ goto fail; - if (size%size_of_a_sample(cfg.cmp_mode) != 0) { - fprintf(stderr, "\n%s: %s: Error: The data file is not correct formatted. Expected multiple of %zu hex words.\n", - PROGRAM_NAME, data_file_name, size_of_a_sample(cfg.cmp_mode)); - goto fail; - } - cfg.samples = size/size_of_a_sample(cfg.cmp_mode); - printf("\nNo samples parameter set. Use samples = %u.\n... ", - cfg.samples); + printf("\nNo samples parameter set. Use samples = %u.\n... ", cfg.samples); } - cfg.input_buf = malloc(cfg.samples * size_of_a_sample(cfg.cmp_mode)); + cfg.input_buf = malloc(cmp_cal_size_of_data(cfg.samples, cfg.cmp_mode)); if (!cfg.input_buf) { fprintf(stderr, "%s: Error allocating memory for input data buffer.\n", PROGRAM_NAME); goto fail; @@ -290,51 +325,81 @@ int main(int argc, char **argv) verbose_en); if (size < 0) goto fail; - else - printf("DONE\n"); + printf("DONE\n"); } else { /* decompression mode*/ - uint32_t cmp_size_byte; - - /* printf("### Decompression ###\n\n"); */ - printf("Importing decompression information file %s ... ", info_file_name); + printf("## Starting the decompression ##\n"); + if (info_file_name) { + ssize_t size; + uint32_t cmp_size_byte; - error = read_cmp_info(info_file_name, &info, verbose_en); - if (error) - goto fail; - else + printf("Importing decompression information file %s ... ", info_file_name); + error = read_cmp_info(info_file_name, &info, verbose_en); + if (error) + goto fail; printf("DONE\n"); - printf("Importing compressed data file %s ... ", data_file_name); - /* cmp_size unit is in bits */ - cmp_size_byte = (info.cmp_size + 31) / 8; - decomp_input_buf = malloc(cmp_size_byte); - if (!decomp_input_buf) { - fprintf(stderr, "%s: Error allocating memory for decompression input buffer.\n", PROGRAM_NAME); - goto fail; - } + printf("Importing compressed data file %s ... ", data_file_name); + cmp_size_byte = cmp_bit_to_4byte(info.cmp_size); + cmp_data_adr = decomp_input_buf = malloc(cmp_size_byte); + if (!decomp_input_buf) { + fprintf(stderr, "%s: Error allocating memory for decompression input buffer.\n", PROGRAM_NAME); + goto fail; + } - size = read_file32(data_file_name, decomp_input_buf, - cmp_size_byte/4, verbose_en); - if (size < 0) - goto fail; - else + size = read_file32(data_file_name, decomp_input_buf, + cmp_size_byte/4, verbose_en); + if (size < 0) + goto fail; + } else { /* read in compressed data with header */ + ssize_t size; + size_t buf_size; + struct cmp_entity *ent; + + printf("Importing compressed data file %s ... ", data_file_name); + buf_size = size = read_file_cmp_entity(data_file_name, + NULL, 0, 0); + if (size < 0) + goto fail; + /* to be save allocate at least the size of the cmp_entity struct */ + if (buf_size < sizeof(struct cmp_entity)) + buf_size = sizeof(struct cmp_entity); + + decomp_input_buf = ent = malloc(buf_size); + if (!ent) { + fprintf(stderr, "%s: Error allocating memory for the compression entity buffer.\n", PROGRAM_NAME); + goto fail; + } + size = read_file_cmp_entity(data_file_name, ent, size, + verbose_en); + if (size < 0) + goto fail; printf("DONE\n"); + + printf("Parse the compression entity header ... "); + error = cmp_ent_read_imagette_header(ent, &info); + if (error) + goto fail; + cmp_data_adr = cmp_ent_get_data_buf(ent); + if (verbose_en) + print_cmp_info(&info); + } + printf("DONE\n"); } /* read in model */ if ((cmp_operation && model_mode_is_used(cfg.cmp_mode)) || (!cmp_operation && model_mode_is_used(info.cmp_mode_used)) || (guess_operation && model_file_name)) { + ssize_t size; uint32_t model_length; + printf("Importing model file %s ... ", model_file_name ? model_file_name : ""); if (!model_file_name) { fprintf(stderr, "%s: No model file (-m option) specified.\n", PROGRAM_NAME); goto fail; } - printf("Importing model file %s ... ", model_file_name); - if (cmp_operation || guess_operation) model_length = cfg.samples; else @@ -346,26 +411,25 @@ int main(int argc, char **argv) goto fail; } - size = read_file16(model_file_name, input_model_buf, model_length, verbose_en); + size = read_file16(model_file_name, input_model_buf, + model_length, verbose_en); if (size < 0) goto fail; - else - printf("DONE\n"); + printf("DONE\n"); - if (cmp_operation || guess_operation) - cfg.model_buf = input_model_buf; + cfg.model_buf = input_model_buf; } if (guess_operation) { error = guess_cmp_pars(&cfg, guess_cmp_mode, guess_level); if (error) goto fail; - } else if (cmp_operation) { /* perform a compression */ + } else if (cmp_operation) { error = compression(&cfg, &info); if (error) goto fail; - } else { /* perform a decompression */ - error = decompression(decomp_input_buf, input_model_buf, &info); + } else { + error = decompression(cmp_data_adr, input_model_buf, &info); if (error) goto fail; } @@ -374,14 +438,12 @@ int main(int argc, char **argv) if (!guess_operation && ((cmp_operation && model_mode_is_used(cfg.cmp_mode)) || (!cmp_operation && model_mode_is_used(info.cmp_mode_used)))) { - printf("Write updated model to file %s_upmodel.dat ... ", - output_prefix); + printf("Write updated model to file %s_upmodel.dat ... ", output_prefix); error = write_to_file16(input_model_buf, info.samples_used, output_prefix, "_upmodel.dat", verbose_en); if (error) goto fail; - else - printf("DONE\n"); + printf("DONE\n"); } free(cfg.input_buf); @@ -401,40 +463,23 @@ fail: } -/* generate packets to setup a RDCU compression */ -static int gen_rdcu_write_pkts(struct cmp_cfg *cfg) +static enum cmp_ent_data_type cmp_ent_map_cmp_mode_data_type(uint32_t cmp_mode) { - int error; - - error = init_rmap_pkt_to_file(); - if (error) { - fprintf(stderr, "%s: Read RMAP packet config file .rdcu_pkt_mode_cfg failed.\n", - PROGRAM_NAME); - return -1; - } - - if (last_info_file_name) { - /* generation of packets for parallel read/write RDCU setup */ - struct cmp_info last_info = {0}; - - error = read_cmp_info(last_info_file_name, &last_info, verbose_en); - if (error) { - fprintf(stderr, "%s: %s: Importing last decompression information file failed.\n", - PROGRAM_NAME, last_info_file_name); - return -1; - } - - error = gen_rdcu_parallel_pkts(cfg, &last_info); - if (error) - return -1; + switch (cmp_mode) { + case MODE_RAW: + return DATA_TYPE_IMAGETTE; + case MODE_MODEL_ZERO: + case MODE_DIFF_ZERO: + case MODE_MODEL_MULTI: + case MODE_DIFF_MULTI: + if (print_rdcu_cfg) + return DATA_TYPE_IMAGETTE_ADAPTIVE; + else + return DATA_TYPE_IMAGETTE; + default: + printf("No mapping between compression mode and header data type\n!"); + return DATA_TYPE_UNKOWN; } - - /* generation of packets for non-parallel read/write RDCU setup */ - error = gen_write_rdcu_pkts(cfg); - if (error) - return -1; - - return 0; } @@ -467,6 +512,11 @@ static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode, cmp_size = cmp_guess(cfg, guess_level); if (!cmp_size) return -1; + + if (include_cmp_header) + cmp_size = CHAR_BIT * (cmp_bit_to_4byte(cmp_size) + + cmp_ent_cal_hdr_size(cmp_ent_map_cmp_mode_data_type(cfg->cmp_mode))); + printf("DONE\n"); printf("Write the guessed compression configuration to file %s.cfg ... ", output_prefix); @@ -482,13 +532,55 @@ static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode, } +/* generate packets to setup a RDCU compression */ +static int gen_rdcu_write_pkts(struct cmp_cfg *cfg) +{ + int error; + + error = init_rmap_pkt_to_file(); + if (error) { + fprintf(stderr, "%s: Read RMAP packet config file .rdcu_pkt_mode_cfg failed.\n", + PROGRAM_NAME); + return -1; + } + + if (last_info_file_name) { + /* generation of packets for parallel read/write RDCU setup */ + struct cmp_info last_info = {0}; + + error = read_cmp_info(last_info_file_name, &last_info, verbose_en); + if (error) { + fprintf(stderr, "%s: %s: Importing last decompression information file failed.\n", + PROGRAM_NAME, last_info_file_name); + return -1; + } + + error = gen_rdcu_parallel_pkts(cfg, &last_info); + if (error) + return -1; + } + + /* generation of packets for non-parallel read/write RDCU setup */ + error = gen_write_rdcu_pkts(cfg); + if (error) + return -1; + + return 0; +} + + /* compress the data and write the results to files */ static int compression(struct cmp_cfg *cfg, struct cmp_info *info) { int error; uint32_t cmp_size_byte; - - cfg->icu_output_buf = NULL; + uint8_t *out_buf = NULL; + uint32_t out_buf_size; + uint8_t model_counter = DEFAULT_MODEL_COUNTER; + uint16_t model_id = DEFAULT_MODEL_ID; + size_t cmp_hdr_size = 0; + enum cmp_ent_data_type data_type = DATA_TYPE_UNKOWN; + uint64_t start_time = cmp_ent_create_timestamp(NULL); if (cfg->buffer_length == 0) { cfg->buffer_length = (cfg->samples+1) * BUFFER_LENGTH_DEF_FAKTOR; /* +1 to prevent malloc(0)*/ @@ -501,55 +593,92 @@ static int compression(struct cmp_cfg *cfg, struct cmp_info *info) error = gen_rdcu_write_pkts(cfg); if (error) goto error_cleanup; - else - printf("... DONE\n"); + printf("... DONE\n"); } printf("Compress data ... "); + out_buf_size = (cmp_cal_size_of_data(cfg->buffer_length, cfg->cmp_mode) + 3) & ~0x3U; + if (include_cmp_header) { + uint32_t red_val; - cfg->icu_output_buf = malloc(cfg->buffer_length * size_of_a_sample(cfg->cmp_mode)); - if (cfg->icu_output_buf == NULL) { + data_type = cmp_ent_map_cmp_mode_data_type(cfg->cmp_mode); + cmp_hdr_size = cmp_ent_cal_hdr_size(data_type); + if (!cmp_hdr_size) + goto error_cleanup; + + if (model_id_str) { + error = atoui32("model_id", model_id_str, &red_val); + if (error || red_val > UINT16_MAX) + goto error_cleanup; + model_id = red_val; + } + if (model_counter_str) { + error = atoui32("model_counter", model_counter_str, &red_val); + if (error || red_val > UINT8_MAX) + goto error_cleanup; + model_counter = red_val; + } else { + if (model_mode_is_used(cfg->cmp_mode)) + model_counter = DEFAULT_MODEL_COUNTER + 1; + } + } + + + out_buf = malloc(out_buf_size + cmp_hdr_size + 3); + if (out_buf == NULL) { fprintf(stderr, "%s: Error allocating memory for output buffer.\n", PROGRAM_NAME); goto error_cleanup; } + cfg->icu_output_buf = out_buf + cmp_hdr_size; error = icu_compress_data(cfg, info); if (error || info->cmp_err != 0) { printf("\nCompression error 0x%02X\n... ", info->cmp_err); - if ((info->cmp_err >> SMALL_BUFFER_ERR_BIT) & 1U) - fprintf(stderr, "%s: the buffer for the compressed data is too small. Try a larger buffer_length parameter.\n", PROGRAM_NAME); + /* TODO: add a parse cmp error function */ + /* if ((info->cmp_err >> SMALL_BUFFER_ERR_BIT) & 1U) */ + /* fprintf(stderr, "%s: the buffer for the compressed data is too small. Try a larger buffer_length parameter.\n", PROGRAM_NAME); */ goto error_cleanup; - } else - printf("DONE\n"); + } + if (include_cmp_header) { + struct cmp_entity *ent = (struct cmp_entity *)out_buf; + size_t s = cmp_ent_build(ent, data_type, cmp_tool_gen_version_id(VERSION), + start_time, cmp_ent_create_timestamp(NULL), + model_id, model_counter, info, cfg); + if (!s) { + fprintf(stderr, "%s: error occurred while creating the compression entity header.\n", PROGRAM_NAME); + goto error_cleanup; + } + } + printf("DONE\n"); if (rdcu_pkt_mode) { printf("Generate the read results packets ... "); error = gen_read_rdcu_pkts(info); if (error) goto error_cleanup; - else - printf("DONE\n"); + printf("DONE\n"); } printf("Write compressed data to file %s.cmp ... ", output_prefix); - /* length of cmp_size in bytes words (round up to 4 bytes) */ - cmp_size_byte = (info->cmp_size + 31)/32 * 4; - error = write_cmp_data_file(cfg->icu_output_buf, cmp_size_byte, - output_prefix, ".cmp", verbose_en); - if (error) - goto error_cleanup; + if (include_cmp_header) + cmp_size_byte = cmp_ent_get_size((struct cmp_entity *)out_buf); else - printf("DONE\n"); - free(cfg->icu_output_buf); - cfg->icu_output_buf = NULL; + cmp_size_byte = cmp_bit_to_4byte(info->cmp_size); - printf("Write decompression information to file %s.info ... ", - output_prefix); - error = write_info(info, output_prefix, print_rdcu_cfg); + error = write_cmp_data_file(out_buf, cmp_size_byte, output_prefix, + ".cmp", verbose_en); if (error) goto error_cleanup; - else + printf("DONE\n"); + + if (!include_cmp_header) { + printf("Write decompression information to file %s.info ... ", + output_prefix); + error = write_info(info, output_prefix, print_rdcu_cfg); + if (error) + goto error_cleanup; printf("DONE\n"); + } if (verbose_en) { printf("\n"); @@ -557,16 +686,19 @@ static int compression(struct cmp_cfg *cfg, struct cmp_info *info) printf("\n"); } + free(out_buf); + out_buf = NULL; + return 0; error_cleanup: - free(cfg->icu_output_buf); + free(out_buf); return -1; } /* decompress the data and write the results in file(s)*/ -static int decompression(uint32_t *decomp_input_buf, uint16_t *input_model_buf, +static int decompression(uint32_t *cmp_data_adr, uint16_t *input_model_buf, struct cmp_info *info) { int error; @@ -574,17 +706,20 @@ static int decompression(uint32_t *decomp_input_buf, uint16_t *input_model_buf, printf("Decompress data ... "); - if (info->samples_used == 0) - return 0; /* nothing to decompress */ + if (info->samples_used == 0) { + printf("\nWarring: No data are decompressed.\n... "); + printf("DONE\n"); + return 0; + } - decomp_output = malloc(info->samples_used * - size_of_a_sample(info->cmp_mode_used)); + decomp_output = malloc(cmp_cal_size_of_data(info->samples_used, + info->cmp_mode_used)); if (decomp_output == NULL) { fprintf(stderr, "%s: Error allocating memory for decompressed data.\n", PROGRAM_NAME); return -1; } - error = decompress_data(decomp_input_buf, input_model_buf, info, + error = decompress_data(cmp_data_adr, input_model_buf, info, decomp_output); if (error) { free(decomp_output); diff --git a/how_to_no_header.md b/how_to_no_header.md new file mode 100644 index 0000000000000000000000000000000000000000..3d0f8004ca96299be5f19225f66c7065534bd622 --- /dev/null +++ b/how_to_no_header.md @@ -0,0 +1,55 @@ +## How to use the tool without header + +A simple example to show how the compression tool works without the compression entity header. + +0. Download the [tool][3] or run `make` to build the tool + +1. Create a configuration file +* Create a cfg directory + `mkdir cfg` +* To create a default 1d-differencing mode configuration: + `./cmp_tool --diff_cfg > cfg/default_config_1d.cfg` +* To create a default model mode configuration: + `./cmp_tool --model_cfg > cfg/default_config_model.cfg` +* Change the the **`samples`** and **`buffer_length`** parameters from `0` to `50` +in the `default_config_1d.cfg` and `default_config_model.cfg` files + +2. Compress data with the default configurations: +* Create a directory for the compressed data + `mkdir compressed` +* 1d-differencing mode compression + `./cmp_tool -c cfg/default_config_1d.cfg -d test_data/test_data1.dat --no_header -o compressed/data1` + This creates these two files in the compressed directory: + `data1.cmp` -> compressed `test_data1.dat` file + `data1.info` -> decompression information for `data1.cmp` +* Model mode compression + `./cmp_tool -c cfg/default_config_model.cfg -d test_data/test_data2.dat -m test_data/test_data1.dat --no_header -o compressed/data2` + We use `test_data1.dat` as the first model for `test_data2.dat`. + + Creates these three files in the compressed directory: + `data2.cmp ` -> compressed `test_data3.dat` file + `data2.info` -> decompression information for `data2.cmp` + `data2_upmodel.dat` -> updated model used to **compress** the next data in model mode + +3. Decompress the data +* Create a directory for the decompressed data + `mkdir decompressed` +* Decompress `data1.cmp` + `./cmp_tool -i compressed/data1.info -d compressed/data1.cmp -o decompressed/test_data1` + Creates this file in the decompressed directory: + `test_data1.dat ` -> decompressed `data1.cmp` file +* Decompress `data2.cmp` + `./cmp_tool -i compressed/data2.info -d compressed/data2.cmp -m decompressed/test_data1.dat -o decompressed/test_data2` + As for the compression we use `test_data1.dat` as our model for decompression. + + Creates these two files in the decompressed directory: + `test_data2.dat ` -> decompressed `data2.cmp` file + `data2_upmodel.dat` -> updated model used to **decompress** the next data in model mode + +4. Bonus: Check if the decompressed data are equal to the original data + The moment of truth! + `diff test_data/test_data1.dat decompressed/test_data1.dat` + `diff test_data/test_data2.dat decompressed/test_data2.dat` + + And also check if the updated model is the same + `diff compressed/data2_upmodel.dat decompressed/test_data2_upmodel.dat` \ No newline at end of file diff --git a/include/byteorder.h b/include/byteorder.h index b0397059a8dfb1f632f2df24db380ec1d46beab9..42fa9b850bc7faed218e92790ee04d176265b97f 100644 --- a/include/byteorder.h +++ b/include/byteorder.h @@ -72,14 +72,22 @@ (((uint32_t)(x) & (uint32_t)0x00ff0000UL) >> 8) | \ (((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24))) - -#ifdef USE_BUILTIN_BSWAP -#if GCC_VERSION >= 40400 +#define ___constant_swab64(x) ((uint64_t)( \ + (((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) | \ + (((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) | \ + (((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) | \ + (((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) << 8) | \ + (((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >> 8) | \ + (((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) | \ + (((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) | \ + (((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56))) + +#if GCC_VERSION >= 40400 || defined(__clang__) #define __HAVE_BUILTIN_BSWAP32__ +#define __HAVE_BUILTIN_BSWAP64__ #endif -#if GCC_VERSION >= 40800 +#if GCC_VERSION >= 40800 || defined(__clang__) #define __HAVE_BUILTIN_BSWAP16__ -#endif #endif /* USE_BUILTIN_BSWAP */ @@ -103,6 +111,15 @@ static inline __attribute__((const)) uint32_t __fswab32(uint32_t val) } +static inline __attribute__((const)) uint64_t __fswab64(uint64_t val) +{ +#ifdef __HAVE_BUILTIN_BSWAP64__ + return __builtin_bswap64(val); +#else + return ___constant_swab64(val); +#endif +} + /** * @brief return a byteswapped 16-bit value * @param x value to byteswap @@ -125,6 +142,16 @@ static inline __attribute__((const)) uint32_t __fswab32(uint32_t val) __fswab32(x)) +/** + * @brief return a byteswapped 64-bit value + * @param x a value to byteswap + */ + +#define __swab64(x) \ + (__builtin_constant_p((uint64_t)(x)) ? \ + ___constant_swab64(x) : \ + __fswab64(x)) + /** * @brief return a byteswapped 16-bit value from a pointer * @param p a pointer to a naturally-aligned 16-bit value @@ -145,6 +172,16 @@ static inline uint32_t __swab32p(const uint32_t *p) } +/** + * @brief return a byteswapped 64-bit value from a pointer + * @param p a pointer to a naturally-aligned 64-bit value + */ +static inline uint64_t __swab64p(const uint64_t *p) +{ + return __swab64(*p); +} + + /** * @brief byteswap a 16-bit value in-place * @param p a pointer to a naturally-aligned 16-bit value @@ -167,34 +204,52 @@ static inline void __swab32s(uint32_t *p) } +/** + * @brief byteswap a 64-bit value in-place + * @param p a pointer to a naturally-aligned 64-bit value + */ + +static inline void __swab64s(uint64_t *p) +{ + *p = __swab64p(p); +} + #ifdef __BIG_ENDIAN #define __cpu_to_le16(x) ((uint16_t)__swab16((x))) #define __cpu_to_le32(x) ((uint32_t)__swab32((x))) +#define __cpu_to_le64(x) ((uint64_t)__swab64((x))) #define __cpu_to_le16s(x) __swab16s((x)) #define __cpu_to_le32s(x) __swab32s((x)) +#define __cpu_to_le64s(x) __swab64s((x)) #define __cpu_to_be16(x) ((uint16_t)(x)) #define __cpu_to_be32(x) ((uint32_t)(x)) +#define __cpu_to_be64(x) ((uint64_t)(x)) #define __cpu_to_be16s(x) { (void)(x); } #define __cpu_to_be32s(x) { (void)(x); } +#define __cpu_to_be64s(x) { (void)(x); } #define __le16_to_cpu(x) __swab16((uint16_t)(x)) #define __le32_to_cpu(x) __swab32((uint32_t)(x)) +#define __le64_to_cpu(x) __swab64((uint64_t)(x)) #define __le16_to_cpus(x) __swab16s((x)) #define __le32_to_cpus(x) __swab32s((x)) +#define __le64_to_cpus(x) __swab64s((x)) #define __be16_to_cpu(x) ((uint16_t)(x)) #define __be32_to_cpu(x) ((uint32_t)(x)) +#define __be64_to_cpu(x) ((uint64_t)(x)) #define __be16_to_cpus(x) { (void)(x); } #define __be32_to_cpus(x) { (void)(x); } +#define __be64_to_cpus(x) { (void)(x); } #endif /* __BIG_ENDIAN */ @@ -203,29 +258,37 @@ static inline void __swab32s(uint32_t *p) #define __cpu_to_le16(x) ((uint16_t)(x)) #define __cpu_to_le32(x) ((uint32_t)(x)) +#define __cpu_to_le64(x) ((uint64_t)(x)) #define __cpu_to_le16s(x) { (void)(x); } #define __cpu_to_le32s(x) { (void)(x); } +#define __cpu_to_le64s(x) { (void)(x); } #define __cpu_to_be16(x) ((uint16_t)__swab16((x))) #define __cpu_to_be32(x) ((uint32_t)__swab32((x))) +#define __cpu_to_be64(x) ((uint64_t)__swab64((x))) #define __cpu_to_be16s(x) __swab16s((x)) #define __cpu_to_be32s(x) __swab32s((x)) +#define __cpu_to_be64s(x) __swab64s((x)) #define __le16_to_cpu(x) ((uint16_t)(x)) #define __le32_to_cpu(x) ((uint32_t)(x)) +#define __le64_to_cpu(x) ((uint64_t)(x)) +#define __le64_to_cpus(x) { (void)(x); } #define __le32_to_cpus(x) { (void)(x); } #define __le16_to_cpus(x) { (void)(x); } #define __be16_to_cpu(x) __swab16((uint16_t)(uint16_t)(x)) #define __be32_to_cpu(x) __swab32((uint32_t)(uint32_t)(x)) +#define __be64_to_cpu(x) __swab64((uint64_t)(uint64_t)(x)) #define __be16_to_cpus(x) __swab16s((x)) #define __be32_to_cpus(x) __swab32s((x)) +#define __be64_to_cpus(x) __swab64s((x)) #endif /* __LITTLE_ENDIAN */ @@ -236,18 +299,22 @@ static inline void __swab32s(uint32_t *p) /** convert cpu order to little endian */ #define cpu_to_le16 __cpu_to_le16 #define cpu_to_le32 __cpu_to_le32 +#define cpu_to_le64 __cpu_to_le64 /** in-place convert cpu order to little endian */ #define cpu_to_le16s __cpu_to_le16s #define cpu_to_le32s __cpu_to_le32s +#define cpu_to_le64s __cpu_to_le64s /** convert cpu order to big endian */ #define cpu_to_be16 __cpu_to_be16 #define cpu_to_be32 __cpu_to_be32 +#define cpu_to_be64 __cpu_to_be64 /** in-place convert cpu order to big endian */ #define cpu_to_be16s __cpu_to_be16s #define cpu_to_be32s __cpu_to_be32s +#define cpu_to_be64s __cpu_to_be64s /* same, but in reverse */ @@ -255,18 +322,22 @@ static inline void __swab32s(uint32_t *p) /** convert little endian to cpu order*/ #define le16_to_cpu __le16_to_cpu #define le32_to_cpu __le32_to_cpu +#define le64_to_cpu __le64_to_cpu /** in-place convert little endian to cpu order*/ #define le16_to_cpus __le16_to_cpus #define le32_to_cpus __le32_to_cpus +#define le64_to_cpus __le64_to_cpus /** convert big endian to cpu order*/ #define be16_to_cpu __be16_to_cpu #define be32_to_cpu __be32_to_cpu +#define be64_to_cpu __be64_to_cpu /** in-place convert big endian to cpu order*/ #define be16_to_cpus __be16_to_cpus #define be32_to_cpus __be32_to_cpus +#define be64_to_cpus __be64_to_cpus diff --git a/include/cmp_debug.h b/include/cmp_debug.h index ba345d205c0a9c50b52260de2577e80cdedc16f3..defc4e37da7d10a834cbe471fddff97cad498b8c 100644 --- a/include/cmp_debug.h +++ b/include/cmp_debug.h @@ -21,7 +21,7 @@ #include <stdio.h> -#if defined(DEBUG) || DDEBUGLEVEL > 0 +#if defined(DEBUG) || DEBUGLEVEL > 0 __extension__ #define debug_print(...) \ do { fprintf(stderr, __VA_ARGS__); } while (0) diff --git a/include/cmp_entity.h b/include/cmp_entity.h new file mode 100644 index 0000000000000000000000000000000000000000..46b9ac3b33b7bfc123a87fbce660b95ebd0828d0 --- /dev/null +++ b/include/cmp_entity.h @@ -0,0 +1,318 @@ +/** + * @file cmp_entity.h + * @author Dominik Loidolt (dominik.loidolt@univie.ac.at), + * @date Mai, 2021 + * + * @copyright GPLv2 + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * @brief functions and definition to handle a compression entity + * @see Data Compression User Manual PLATO-UVIE-PL-UM-0001 + * + * @note this code can also used on a little endian machine + * + * @warning: If you create an entity of one data product type and use get/set + * functions intended for another data product type, it will result in a + * corrupted entity. Do not do this. + */ + + +#ifndef CMP_ENTITY_H +#define CMP_ENTITY_H + +#include <stdint.h> + +#include "compiler.h" +#include "cmp_support.h" + + +/* Defined Compression Data Product Types */ +enum cmp_ent_data_type { + DATA_TYPE_IMAGETTE = 1, + DATA_TYPE_IMAGETTE_ADAPTIVE, + DATA_TYPE_SAT_IMAGETTE, + DATA_TYPE_SAT_IMAGETTE_ADAPTIVE, + DATA_TYPE_OFFSET, + DATA_TYPE_BACKGROUND, + DATA_TYPE_SMEARING, + DATA_TYPE_S_FX, + DATA_TYPE_S_FX_DFX, + DATA_TYPE_S_FX_NCOB, + DATA_TYPE_S_FX_DFX_NCOB_ECOB, + DATA_TYPE_L_FX, + DATA_TYPE_L_FX_DFX, + DATA_TYPE_L_FX_NCOB, + DATA_TYPE_L_FX_DFX_NCOB_ECOB, + DATA_TYPE_F_FX, + DATA_TYPE_F_FX_DFX, + DATA_TYPE_F_FX_NCOB, + DATA_TYPE_F_FX_DFX_NCOB_ECOB, + DATA_TYPE_F_CAM_IMAGETTE, + DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE, + DATA_TYPE_F_CAM_OFFSET, + DATA_TYPE_F_CAM_BACKGROUND, + DATA_TYPE_UNKOWN = -1 +}; + +#define GENERIC_HEADER_SIZE 30 +#define SPECIFIC_IMAGETTE_HEADER_SIZE 6 +#define SPECIFIC_IMAGETTE_ADAPTIVE_HEADER_SIZE 10 +#define SPECIFIC_NON_IMAGETTE_HEADER_SIZE 30 /* TBC */ + +#define IMAGETTE_HEADER_SIZE \ + (GENERIC_HEADER_SIZE + SPECIFIC_IMAGETTE_HEADER_SIZE) +#define IMAGETTE_ADAPTIVE_HEADER_SIZE \ + (GENERIC_HEADER_SIZE + SPECIFIC_IMAGETTE_ADAPTIVE_HEADER_SIZE) +#define NON_IMAGETTE_HEADER_SIZE \ + (GENERIC_HEADER_SIZE + SPECIFIC_NON_IMAGETTE_HEADER_SIZE) + +#define CMP_ENTITY_MAX_SIZE 0xFFFFFFUL + +#define RAW_BIT_IN_DATA_TYPE 15U + +#define CMP_TOOL_VERSION_ID_BIT 0x8000U /* TBC */ + +__extension__ +struct timestamp_cmp_ent { + uint32_t coarse; + uint16_t fine; +} __attribute__((packed)); + +__extension__ +struct imagette_header { + uint16_t spill_used; /* Spillover threshold used */ + uint8_t golomb_par_used; /* Golomb parameter used */ + union{ + struct { + uint8_t spare1; + uint16_t spare2; + uint8_t ima_cmp_dat[]; /* compressed data for imagette specific header */ + } __attribute__((packed)); + struct { + uint16_t ap1_spill_used; /* Adaptive Spillover threshold used 1 */ + uint8_t ap1_golomb_par_used; /* Adaptive Golomb parameter used 1 */ + uint16_t ap2_spill_used; /* Adaptive Spillover threshold used 2 */ + uint8_t ap2_golomb_par_used; /* Adaptive Golomb parameter used 2 */ + uint8_t spare3; + uint8_t ap_ima_cmp_data[]; /* compressed data for adaptive imagette specific header */ + } __attribute__((packed)); + }; +} __attribute__((packed)); +compile_time_assert(sizeof(struct imagette_header) == SPECIFIC_IMAGETTE_ADAPTIVE_HEADER_SIZE, AP_IMAGETTE_HEADER_T_SIZE_IS_NOT_CORRECT); + +__extension__ +struct non_imagette_header { + uint32_t spill_1_used:24; /* spillover threshold 1 used */ + uint16_t cmp_par_1_used; /* compression parameter 1 used */ + uint32_t spill_2_used:24; /* spillover threshold 2 used */ + uint16_t cmp_par_2_used; /* compression parameter 2 used */ + uint32_t spill_3_used:24; /* spillover threshold 3 used */ + uint16_t cmp_par_3_used; /* compression parameter 3 used */ + uint32_t spill_4_used:24; /* spillover threshold 4 used */ + uint16_t cmp_par_4_used; /* compression parameter 4 used */ + uint32_t spill_5_used:24; /* spillover threshold 5 used */ + uint16_t cmp_par_5_used; /* compression parameter 5 used */ + uint32_t spill_6_used:24; /* spillover threshold 6 used */ + uint16_t cmp_par_6_used; /* compression parameter 6 used */ + uint8_t cmp_data[]; +} __attribute__((packed)); +compile_time_assert(sizeof(struct non_imagette_header) == SPECIFIC_NON_IMAGETTE_HEADER_SIZE, NON_IMAGETTE_HEADER_T_SIZE_IS_NOT_CORRECT); + +__extension__ +struct cmp_entity { + uint16_t asw_version_id; /* ICU ASW Version ID */ + uint32_t cmp_ent_size:24; /* Compression Entity Size */ + uint32_t original_size:24; /* Original Data Size */ + union { + uint64_t start_timestamp:48; /* Compression Start Timestamp */ + struct timestamp_cmp_ent start_time; + } __attribute__((packed)); + union { + uint64_t end_timestamp:48; /* Compression End Timestamp */ + struct timestamp_cmp_ent end_time; + } __attribute__((packed)); + uint16_t data_type; /* Data Product Type */ + uint8_t cmp_mode_used; /* used Compression Mode */ + uint8_t model_value_used; /* used Model Updating Weighing Value */ + uint16_t model_id; /* Model ID */ + uint8_t model_counter; /* Model Counter */ + uint8_t spare; + uint16_t lossy_cmp_par_used; /* used Lossy Compression Parameters */ + union { /* specific Compression Entity Header for the different Data Product Types */ + struct imagette_header ima; + struct non_imagette_header non_ima; + }; +} __attribute__((packed)); +compile_time_assert(sizeof(struct cmp_entity) == NON_IMAGETTE_HEADER_SIZE, CMP_ENTITY_SIZE_IS_NOT_CORRECT); + + + +/* brief create a compression entity by setting the size of the + * compression entity and the data product type in the entity header + */ +size_t cmp_ent_create(struct cmp_entity *ent, enum cmp_ent_data_type data_type, + uint32_t cmp_size_byte); + +/* create a compression entity and set the header fields */ +size_t cmp_ent_build(struct cmp_entity *ent, enum cmp_ent_data_type data_type, + uint16_t asw_version_id, uint64_t start_time, + uint64_t end_time, uint16_t model_id, uint8_t model_counter, + struct cmp_info *info, struct cmp_cfg *cfg); + +/* read in a imagette compression entity header to a info struct */ +int cmp_ent_read_imagette_header(struct cmp_entity *ent, struct cmp_info *info); + + + +/* set functions for generic compression entity header */ +int cmp_ent_set_asw_version_id(struct cmp_entity *ent, uint32_t asw_version_id); +int cmp_ent_set_size(struct cmp_entity *ent, uint32_t cmp_ent_size); + +int cmp_ent_set_original_size(struct cmp_entity *ent, uint32_t original_size); + +int cmp_ent_set_start_timestamp(struct cmp_entity *ent, uint64_t start_timestamp); +int cmp_ent_set_coarse_start_time(struct cmp_entity *ent, uint32_t coarse_time); +int cmp_ent_set_fine_start_time(struct cmp_entity *ent, uint16_t fine_time); + +int cmp_ent_set_end_timestamp(struct cmp_entity *ent, uint64_t end_timestamp); +int cmp_ent_set_coarse_end_time(struct cmp_entity *ent, uint32_t coarse_time); +int cmp_ent_set_fine_end_time(struct cmp_entity *ent, uint16_t fine_time); + +int cmp_ent_set_data_type(struct cmp_entity *ent, + enum cmp_ent_data_type data_type); +int cmp_ent_set_data_type_raw_bit(struct cmp_entity *ent, int raw_bit); +int cmp_ent_set_cmp_mode(struct cmp_entity *ent, uint32_t cmp_mode_used); +int cmp_ent_set_model_value(struct cmp_entity *ent, uint32_t model_value_used); +int cmp_ent_set_model_id(struct cmp_entity *ent, uint32_t model_id); +int cmp_ent_set_model_counter(struct cmp_entity *ent, uint32_t model_counter); +int cmp_ent_set_lossy_cmp_par(struct cmp_entity *ent, uint32_t lossy_cmp_par_used); + + +/* set functions for specific entity header for imagette and adaptive imagette + * data product types + */ +int cmp_ent_set_ima_spill(struct cmp_entity *ent, uint32_t spill_used); +int cmp_ent_set_ima_golomb_par(struct cmp_entity *ent, uint32_t golomb_par_used); + + +/* set functions for specific entity header for adaptive imagette data product + * types + */ +int cmp_ent_set_ima_ap1_spill(struct cmp_entity *ent, uint32_t ap1_spill_used); +int cmp_ent_set_ima_ap1_golomb_par(struct cmp_entity *ent, uint32_t ap1_golomb_par_used); + +int cmp_ent_set_ima_ap2_spill(struct cmp_entity *ent, uint32_t ap2_spill_used); +int cmp_ent_set_ima_ap2_golomb_par(struct cmp_entity *ent, uint32_t ap2_golomb_par_used); + + +/* set functions for specific entity header for non-imagette data product types */ +int cmp_ent_set_non_ima_spill1(struct cmp_entity *ent, uint32_t spill1_used); +int cmp_ent_set_non_ima_cmp_par1(struct cmp_entity *ent, uint32_t cmp_par1_used); + +int cmp_ent_set_non_ima_spill2(struct cmp_entity *ent, uint32_t spill2_used); +int cmp_ent_set_non_ima_cmp_par2(struct cmp_entity *ent, uint32_t cmp_par2_used); + +int cmp_ent_set_non_ima_spill3(struct cmp_entity *ent, uint32_t spill3_used); +int cmp_ent_set_non_ima_cmp_par3(struct cmp_entity *ent, uint32_t cmp_par3_used); + +int cmp_ent_set_non_ima_spill4(struct cmp_entity *ent, uint32_t spill4_used); +int cmp_ent_set_non_ima_cmp_par4(struct cmp_entity *ent, uint32_t cmp_par4_used); + +int cmp_ent_set_non_ima_spill5(struct cmp_entity *ent, uint32_t spill5_used); +int cmp_ent_set_non_ima_cmp_par5(struct cmp_entity *ent, uint32_t cmp_par5_used); + +int cmp_ent_set_non_ima_spill6(struct cmp_entity *ent, uint32_t spill6_used); +int cmp_ent_set_non_ima_cmp_par6(struct cmp_entity *ent, uint32_t cmp_par6_used); + + + +/* get functions for generic compression entity header */ +uint16_t cmp_ent_get_asw_version_id(struct cmp_entity *ent); +uint32_t cmp_ent_get_size(struct cmp_entity *ent); +uint32_t cmp_ent_get_original_size(struct cmp_entity *ent); + +uint64_t cmp_ent_get_start_timestamp(struct cmp_entity *ent); +uint32_t cmp_ent_get_coarse_start_time(struct cmp_entity *ent); +uint16_t cmp_ent_get_fine_start_time(struct cmp_entity *ent); + +uint64_t cmp_ent_get_end_timestamp(struct cmp_entity *ent); +uint32_t cmp_ent_get_coarse_end_time(struct cmp_entity *ent); +uint16_t cmp_ent_get_fine_end_time(struct cmp_entity *ent); + +enum cmp_ent_data_type cmp_ent_get_data_type(struct cmp_entity *ent); +int cmp_ent_get_data_type_raw_bit(struct cmp_entity *ent); +uint8_t cmp_ent_get_cmp_mode(struct cmp_entity *ent); +uint8_t cmp_ent_get_model_value_used(struct cmp_entity *ent); + +uint16_t cmp_ent_get_model_id(struct cmp_entity *ent); +uint8_t cmp_ent_get_model_counter(struct cmp_entity *ent); +uint16_t cmp_ent_get_lossy_cmp_par(struct cmp_entity *ent); + + +/* get functions for specific entity header for imagette and adaptive imagette + * data product types + */ +uint16_t cmp_ent_get_ima_spill(struct cmp_entity *ent); +uint8_t cmp_ent_get_ima_golomb_par(struct cmp_entity *ent); + + +/* get functions for specific entity header for adaptive imagette data product + * types + */ +uint16_t cmp_ent_get_ima_ap1_spill(struct cmp_entity *ent); +uint8_t cmp_ent_get_ima_ap1_golomb_par(struct cmp_entity *ent); + +uint16_t cmp_ent_get_ima_ap2_spill(struct cmp_entity *ent); +uint8_t cmp_ent_get_ima_ap2_golomb_par(struct cmp_entity *ent); + + +/* get functions for specific entity header for non-imagette data product types */ +uint32_t cmp_ent_get_non_ima_spill1(struct cmp_entity *ent); +uint16_t cmp_ent_get_non_ima_cmp_par1(struct cmp_entity *ent); + +uint32_t cmp_ent_get_non_ima_spill2(struct cmp_entity *ent); +uint16_t cmp_ent_get_non_ima_cmp_par2(struct cmp_entity *ent); + +uint32_t cmp_ent_get_non_ima_spill3(struct cmp_entity *ent); +uint16_t cmp_ent_get_non_ima_cmp_par3(struct cmp_entity *ent); + +uint32_t cmp_ent_get_non_ima_spill4(struct cmp_entity *ent); +uint16_t cmp_ent_get_non_ima_cmp_par4(struct cmp_entity *ent); + +uint32_t cmp_ent_get_non_ima_spill5(struct cmp_entity *ent); +uint16_t cmp_ent_get_non_ima_cmp_par5(struct cmp_entity *ent); + +uint32_t cmp_ent_get_non_ima_spill6(struct cmp_entity *ent); +uint16_t cmp_ent_get_non_ima_cmp_par6(struct cmp_entity *ent); + + +/* get function for the compressed data buffer in the entity */ +void *cmp_ent_get_data_buf(struct cmp_entity *ent); +uint32_t cmp_ent_get_cmp_data_size(struct cmp_entity *ent); + +/* calculate the size of the compression entity header */ +uint32_t cmp_ent_cal_hdr_size(enum cmp_ent_data_type data_type); + + +#if __has_include(<time.h>) +#include <time.h> +/* create a timestamp for the compression header */ +extern const struct tm EPOCH_DATE; +uint64_t cmp_ent_create_timestamp(const struct timespec *ts); +#endif + +/* print and parse functions */ +void cmp_ent_print_header(struct cmp_entity *ent); +void cmp_ent_print_data(struct cmp_entity *ent); +void cmp_ent_print(struct cmp_entity *ent); + +void cmp_ent_parse(struct cmp_entity *ent); + +#endif /* CMP_ENTITY_H */ diff --git a/include/cmp_support.h b/include/cmp_support.h index aeafd0df6209cc39b14ed8648ad06d69f748fe29..d2387ba9dce805cda7cff043dc18fb26c60dd746 100644 --- a/include/cmp_support.h +++ b/include/cmp_support.h @@ -179,8 +179,9 @@ uint32_t get_max_spill(unsigned int golomb_par, unsigned int cmp_mode); uint32_t cmp_get_good_spill(unsigned int golomb_par, unsigned int cmp_mode); size_t size_of_a_sample(unsigned int cmp_mode); -unsigned int size_of_bitstream(unsigned int cmp_size); -unsigned int size_of_model(unsigned int samples, unsigned int cmp_mode); +unsigned int cmp_bit_to_4byte(unsigned int cmp_size_bit); +unsigned int cmp_cal_size_of_data(unsigned int samples, unsigned int cmp_mode); + void print_cmp_cfg(const struct cmp_cfg *cfg); void print_cmp_info(const struct cmp_info *info); diff --git a/include/cmp_tool_lib.h b/include/cmp_tool_lib.h index b8eadaadd57c93fdf3f4e661bdf3db575f4bba09..40ad0be5961ab721950ca831fd330595a2dc1494 100644 --- a/include/cmp_tool_lib.h +++ b/include/cmp_tool_lib.h @@ -18,6 +18,7 @@ #include <string.h> #include "cmp_support.h" +#include "cmp_entity.h" #define PROGRAM_NAME "cmp_tool" #define MAX_CONFIG_LINE 256 @@ -37,6 +38,10 @@ ssize_t read_file16(const char *file_name, uint16_t *buf, uint32_t samples, int verbose_en); ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t samples, int verbose_en); +ssize_t read_file_cmp_entity(const char *file_name, struct cmp_entity *ent, + uint32_t ent_size, int verbose_en); + +uint16_t cmp_tool_gen_version_id(const char *version); int write_cmp_data_file(const void *buf, uint32_t buf_size, const char *output_prefix, const char *name_extension, int verbose); @@ -48,4 +53,5 @@ int write_cfg(const struct cmp_cfg *cfg, const char *output_prefix, int rdcu_cfg int verbose); void print_cfg(const struct cmp_cfg *cfg, int rdcu_cfg); -uint32_t cmp_mode_parse(const char *cmp_mode_str, uint32_t *cmp_mode); +int atoui32(const char *dep_str, const char *val_str, uint32_t *red_val); +int cmp_mode_parse(const char *cmp_mode_str, uint32_t *cmp_mode); diff --git a/include/compiler.h b/include/compiler.h new file mode 100644 index 0000000000000000000000000000000000000000..7bf73afa39daf2f0de032d92479217142f525715 --- /dev/null +++ b/include/compiler.h @@ -0,0 +1,46 @@ +/** + * @file compiler.h + * @author Armin Luntzer (armin.luntzer@univie.ac.at), + * @date 2015 + * + * @copyright GPLv2 + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * @brief a collection of preprocessor macros + */ + + +#ifndef COMPILER_H +#define COMPILER_H + + +/** + * Compile time check usable outside of function scope. + * Stolen from Linux (hpi_internal.h) + */ +#define compile_time_assert(cond, msg) typedef char ASSERT_##msg[(cond) ? 1 : -1] + + +/** + * same with the stuff below + */ + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + + +#define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0])) + +/* optimisation barrier */ +#define barrier() __asm__ __volatile__("": : :"memory") + +#define cpu_relax() barrier() + +#endif diff --git a/lib/cmp_entity.c b/lib/cmp_entity.c new file mode 100644 index 0000000000000000000000000000000000000000..df0c3052867796aff0f23fbd6b70f2216e0131c7 --- /dev/null +++ b/lib/cmp_entity.c @@ -0,0 +1,2294 @@ +/** + * @file cmp_entity.c + * @author Dominik Loidolt (dominik.loidolt@univie.ac.at), + * @date May 2021 + * + * @copyright GPLv2 + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * @brief functions and definition to handle a compression entity + * @see Data Compression User Manual PLATO-UVIE-PL-UM-0001 + */ + +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <limits.h> +#if __has_include(<time.h>) + #include <time.h> + #include <stdlib.h> +#endif + +#include "../include/cmp_entity.h" +#include "../include/cmp_support.h" +#include "../include/byteorder.h" + + +#if __has_include(<time.h>) +/* Used as epoch Wed Jan 1 00:00:00 2020 */ +# if defined(_WIN32) || defined(_WIN64) +const struct tm EPOCH_DATE = { 0, 0, 0, 1, 0, 120, 0, 0, 0, }; +# else +const struct tm EPOCH_DATE = { 0, 0, 0, 1, 0, 120, 0, 0, 0, 0, NULL }; +# endif /* _WIN */ +#endif /* time.h */ + + +/** + * @brief calculate the size of the compression entity header based of the data + * product type + * + * @param data_type compression entity data product type + * + * @returns size of the compression entity header in bytes, 0 on unknown data + * type + */ + +uint32_t cmp_ent_cal_hdr_size(enum cmp_ent_data_type data_type) +{ + switch (data_type) { + case DATA_TYPE_IMAGETTE: + case DATA_TYPE_F_CAM_IMAGETTE: + return IMAGETTE_HEADER_SIZE; + case DATA_TYPE_IMAGETTE_ADAPTIVE: + case DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE: + return IMAGETTE_ADAPTIVE_HEADER_SIZE; + case DATA_TYPE_SAT_IMAGETTE: + case DATA_TYPE_SAT_IMAGETTE_ADAPTIVE: + case DATA_TYPE_OFFSET: + case DATA_TYPE_BACKGROUND: + case DATA_TYPE_SMEARING: + case DATA_TYPE_S_FX: + case DATA_TYPE_S_FX_DFX: + case DATA_TYPE_S_FX_NCOB: + case DATA_TYPE_S_FX_DFX_NCOB_ECOB: + case DATA_TYPE_L_FX: + case DATA_TYPE_L_FX_DFX: + case DATA_TYPE_L_FX_NCOB: + case DATA_TYPE_L_FX_DFX_NCOB_ECOB: + case DATA_TYPE_F_FX: + case DATA_TYPE_F_FX_DFX: + case DATA_TYPE_F_FX_NCOB: + case DATA_TYPE_F_FX_DFX_NCOB_ECOB: + case DATA_TYPE_F_CAM_OFFSET: + case DATA_TYPE_F_CAM_BACKGROUND: + return NON_IMAGETTE_HEADER_SIZE; + case DATA_TYPE_UNKOWN: + return 0; + } + + return 0; +} + + +/** + * @brief set ICU ASW Version ID in the compression entity header + * + * @param ent pointer to a compression entity + * @param asw_version_id the applications software version identifier + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_asw_version_id(struct cmp_entity *ent, uint32_t asw_version_id) +{ + if (!ent) + return -1; + + if (asw_version_id > UINT16_MAX) + return -1; + + ent->asw_version_id = cpu_to_be16(asw_version_id); + + return 0; +} + + +/** + * @brief set the size of the compression entity in the entity header + * + * @param ent pointer to a compression entity + * @param cmp_ent_size the compression entity size measured in bytes + * + * @note maximum size is 2^24-1 + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_size(struct cmp_entity *ent, uint32_t cmp_ent_size) +{ + if (!ent) + return -1; + + if (cmp_ent_size > CMP_ENTITY_MAX_SIZE) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->cmp_ent_size = cpu_to_be32(cmp_ent_size) >> 8; +#else + ent->cmp_ent_size = cmp_ent_size; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set the original size of the compressed data in the entity header + * + * @param ent pointer to a compression entity + * @param original_size the original size of the compressed data measured in + * bytes + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_original_size(struct cmp_entity *ent, uint32_t original_size) +{ + if (!ent) + return -1; + + if (original_size > 0xFFFFFFUL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->original_size = cpu_to_be32(original_size) >> 8; +#else + ent->original_size = original_size; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set the compression start timestamp in the compression entity header + * + * @param ent pointer to a compression entity + * @param start_timestamp compression start timestamp (coarse and fine) + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_start_timestamp(struct cmp_entity *ent, uint64_t start_timestamp) +{ + if (!ent) + return -1; + + if (start_timestamp > 0xFFFFFFFFFFFFULL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->start_timestamp = cpu_to_be64(start_timestamp) >> 16; +#else + ent->start_timestamp = start_timestamp; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set the coarse time in the compression start timestamp in the + * compression entity header + * + * @param ent pointer to a compression entity + * @param coarse_time coarse part of the compression start timestamp + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_coarse_start_time(struct cmp_entity *ent, uint32_t coarse_time) +{ + if (!ent) + return -1; + + ent->start_time.coarse = cpu_to_be32(coarse_time); + + return 0; +} + + +/** + * @brief set the fine time in the compression start timestamp in the + * compression entity header + * + * @param ent pointer to a compression entity + * @param fine_time fine part of the compression start timestamp + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_fine_start_time(struct cmp_entity *ent, uint16_t fine_time) +{ + if (!ent) + return -1; + + ent->start_time.fine = cpu_to_be16(fine_time); + + return 0; +} + + +/** + * @brief set the compression end timestamp in the compression entity header + * + * @param ent pointer to a compression entity + * @param end_timestamp compression end timestamp (coarse and fine) + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_end_timestamp(struct cmp_entity *ent, uint64_t end_timestamp) +{ + if (!ent) + return -1; + + if (end_timestamp > 0xFFFFFFFFFFFFULL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->end_timestamp = cpu_to_be64(end_timestamp) >> 16; +#else + ent->end_timestamp = end_timestamp; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set the coarse time in the compression end timestamp in the + * compression entity header + * + * @param ent pointer to a compression entity + * @param coarse_time coarse part of the compression end timestamp + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_coarse_end_time(struct cmp_entity *ent, uint32_t coarse_time) +{ + if (!ent) + return -1; + + ent->end_time.coarse = cpu_to_be32(coarse_time); + + return 0; +} + + +/** + * @brief set the fine time in the compression end timestamp in the compression + * entity header + * + * @param ent pointer to a compression entity + * @param fine_time fine part of the compression end timestamp + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_fine_end_time(struct cmp_entity *ent, uint16_t fine_time) +{ + if (!ent) + return -1; + + ent->end_time.fine = cpu_to_be16(fine_time); + + return 0; +} + + +/** + * @brief set the data product type in the compression entity header + * + * @param ent pointer to a compression entity + * @param data_type compression entity data product type + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_data_type(struct cmp_entity *ent, + enum cmp_ent_data_type data_type) +{ + if (!ent) + return -1; + + if ((unsigned int)data_type > 0x7FFF) + return -1; + + data_type |= cmp_ent_get_data_type_raw_bit(ent) << RAW_BIT_IN_DATA_TYPE; + + ent->data_type = cpu_to_be16(data_type); + + return 0; +} + + +/** + * @brief set the raw bit in the data product field of the compression entity header + * + * @param ent pointer to a compression entity + * @param raw_bit raw bit is set if raw_bit is non zero + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_data_type_raw_bit(struct cmp_entity *ent, int raw_bit) +{ + uint16_t data_type; + + if (!ent) + return -1; + + if (raw_bit) + data_type = cpu_to_be16(ent->data_type) | 1UL << RAW_BIT_IN_DATA_TYPE; + else + data_type = cpu_to_be16(ent->data_type) & ~(1UL << RAW_BIT_IN_DATA_TYPE); + + ent->data_type = cpu_to_be16(data_type); + + return 0; +} + + +/** + * @brief set the used compression mode in the compression entity header + * + * @param ent pointer to a compression entity + * @param cmp_mode_used used compression mode parameter + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_cmp_mode(struct cmp_entity *ent, uint32_t cmp_mode_used) +{ + if (!ent) + return -1; + + if (cmp_mode_used > UINT8_MAX) + return -1; + + ent->cmp_mode_used = cmp_mode_used; + + return 0; +} + + +/** + * @brief set the used model value in the compression entity header + * + * @param ent pointer to a compression entity + * @param model_value_used used model weighting value parameter + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_model_value(struct cmp_entity *ent, uint32_t model_value_used) +{ + if (!ent) + return -1; + + if (model_value_used > UINT8_MAX) + return -1; + + ent->model_value_used = model_value_used; + + return 0; +} + + +/** + * @brief set model id in the compression entity header + * + * @param ent pointer to a compression entity + * @param model_id the model identifier + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_model_id(struct cmp_entity *ent, uint32_t model_id) +{ + if (!ent) + return -1; + + if (model_id > UINT16_MAX) + return -1; + + ent->model_id = cpu_to_be16(model_id); + + return 0; +} + + +/** + * @brief set model counter in the compression entity header + * + * @param ent pointer to a compression entity + * @param model_counter the model counter + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_model_counter(struct cmp_entity *ent, uint32_t model_counter) +{ + if (!ent) + return -1; + + if (model_counter > UINT8_MAX) + return -1; + + ent->model_counter = model_counter; + + return 0; +} + + +/** + * @brief set the used lossy compression parameter in the compression entity + * header + * + * @param ent pointer to a compression entity + * @param lossy_cmp_par_used used lossy compression parameter + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_lossy_cmp_par(struct cmp_entity *ent, uint32_t lossy_cmp_par_used) +{ + if (!ent) + return -1; + + if (lossy_cmp_par_used > UINT16_MAX) + return -1; + + ent->lossy_cmp_par_used = cpu_to_be16((uint16_t)lossy_cmp_par_used); + + return 0; +} + + +/** + * @brief set the used spillover threshold parameter in the (adaptive) imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * @param spill_used used spillover threshold for imagette data_type + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_ima_spill(struct cmp_entity *ent, uint32_t spill_used) +{ + if (!ent) + return -1; + + if (spill_used > UINT16_MAX) + return -1; + + ent->ima.spill_used = cpu_to_be16(spill_used); + + return 0; +} + + +/** + * @brief set used Golomb parameter in the (adaptive) imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param golomb_par_used used Golomb parameter used for imagette + * data_type + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_ima_golomb_par(struct cmp_entity *ent, uint32_t golomb_par_used) +{ + if (!ent) + return -1; + + if (golomb_par_used > UINT8_MAX) + return -1; + + ent->ima.golomb_par_used = golomb_par_used; + + return 0; +} + + +/* + * @brief set the used adaptive 1 spillover threshold parameter in the adaptive + * imagette specific compression entity header + * + * @param ent pointer to a compression entity + * @param ap1_spill_used used adaptive 1 spillover threshold used for + * semi-adaptive compression feature + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_ima_ap1_spill(struct cmp_entity *ent, uint32_t ap1_spill_used) +{ + if (!ent) + return -1; + + if (ap1_spill_used > UINT16_MAX) + return -1; + + ent->ima.ap1_spill_used = cpu_to_be16(ap1_spill_used); + + return 0; +} + + +/** + * @brief set adaptive 1 Golomb parameter in the adaptive imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param ap1_golomb_par_used used adaptive 1 Golomb parameter for + * semi-adaptive compression feature + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_ima_ap1_golomb_par(struct cmp_entity *ent, uint32_t ap1_golomb_par_used) +{ + if (!ent) + return -1; + + if (ap1_golomb_par_used > UINT8_MAX) + return -1; + + ent->ima.ap1_golomb_par_used = ap1_golomb_par_used; + + return 0; +} + + +/* + * @brief set the used adaptive 2 spillover threshold parameter in the adaptive + * imagette specific compression entity header + * + * @param ent pointer to a compression entity + * @param ap2_spill_used used adaptive 2 spillover threshold used for + * semi-adaptive compression feature + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_ima_ap2_spill(struct cmp_entity *ent, uint32_t ap2_spill_used) +{ + if (!ent) + return -1; + + if (ap2_spill_used > UINT16_MAX) + return -1; + + ent->ima.ap2_spill_used = cpu_to_be16(ap2_spill_used); + + return 0; +} + + +/** + * @brief set adaptive 2 Golomb parameter in the adaptive imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param ap2_golomb_par_used used adaptive 2 Golomb parameter for + * semi-adaptive compression feature + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_ima_ap2_golomb_par(struct cmp_entity *ent, uint32_t ap2_golomb_par_used) +{ + if (!ent) + return -1; + + if (ap2_golomb_par_used > UINT8_MAX) + return -1; + + ent->ima.ap2_golomb_par_used = ap2_golomb_par_used; + + return 0; +} + + +/* + * @brief set the used spillover threshold 1 parameter in the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * @param spill_1_used used spillover threshold parameter 1 for non-imagette + * data_types + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_spill1(struct cmp_entity *ent, uint32_t spill_1_used) +{ + if (!ent) + return -1; + + if (spill_1_used > 0xFFFFFFUL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->non_ima.spill_1_used = cpu_to_be32(spill_1_used) >> 8; +#else + ent->non_ima.spill_1_used = spill_1_used; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set used compression parameter 1 in the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param cmp_par_1_used used compression parameter 1 + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_cmp_par1(struct cmp_entity *ent, uint32_t cmp_par_1_used) +{ + if (!ent) + return -1; + + if (cmp_par_1_used > UINT16_MAX) + return -1; + + ent->non_ima.cmp_par_1_used = cpu_to_be16(cmp_par_1_used); + + return 0; +} + + +/* + * @brief set the used spillover threshold 2 parameter in the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * @param spill_2_used used spillover threshold parameter 2 for non-imagette + * data_types + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_spill2(struct cmp_entity *ent, uint32_t spill_2_used) +{ + if (!ent) + return -1; + + if (spill_2_used > 0xFFFFFFUL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->non_ima.spill_2_used = cpu_to_be32(spill_2_used) >> 8; +#else + ent->non_ima.spill_2_used = spill_2_used; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set used compression parameter 2 in the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param cmp_par_2_used used compression parameter 2 + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_cmp_par2(struct cmp_entity *ent, uint32_t cmp_par_2_used) +{ + if (!ent) + return -1; + + if (cmp_par_2_used > UINT16_MAX) + return -1; + + ent->non_ima.cmp_par_2_used = cpu_to_be16(cmp_par_2_used); + + return 0; +} + + +/* + * @brief set the used spillover threshold 3 parameter in the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * @param spill_3_used used spillover threshold parameter 3 for non-imagette + * data_types + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_spill3(struct cmp_entity *ent, uint32_t spill_3_used) +{ + if (!ent) + return -1; + + if (spill_3_used > 0xFFFFFFUL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->non_ima.spill_3_used = cpu_to_be32(spill_3_used) >> 8; +#else + ent->non_ima.spill_3_used = spill_3_used; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set used compression parameter 3 in the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param cmp_par_3_used used compression parameter 3 + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_cmp_par3(struct cmp_entity *ent, uint32_t cmp_par_3_used) +{ + if (!ent) + return -1; + + if (cmp_par_3_used > UINT16_MAX) + return -1; + + ent->non_ima.cmp_par_3_used = cpu_to_be16(cmp_par_3_used); + + return 0; +} + + +/* + * @brief set the used spillover threshold 4 parameter in the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * @param spill_4_used used spillover threshold parameter 4 for non-imagette + * data_types + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_spill4(struct cmp_entity *ent, uint32_t spill_4_used) +{ + if (!ent) + return -1; + + if (spill_4_used > 0xFFFFFFUL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->non_ima.spill_4_used = cpu_to_be32(spill_4_used) >> 8; +#else + ent->non_ima.spill_4_used = spill_4_used; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set used compression parameter 4 in the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param cmp_par_4_used used compression parameter 4 + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_cmp_par4(struct cmp_entity *ent, uint32_t cmp_par_4_used) +{ + if (!ent) + return -1; + + if (cmp_par_4_used > UINT16_MAX) + return -1; + + ent->non_ima.cmp_par_4_used = cpu_to_be16(cmp_par_4_used); + + return 0; +} + + +/* + * @brief set the used spillover threshold 5 parameter in the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * @param spill_5_used used spillover threshold parameter 5 for non-imagette + * data_types + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_spill5(struct cmp_entity *ent, uint32_t spill_5_used) +{ + if (!ent) + return -1; + + if (spill_5_used > 0xFFFFFFUL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->non_ima.spill_5_used = cpu_to_be32(spill_5_used) >> 8; +#else + ent->non_ima.spill_5_used = spill_5_used; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set used compression parameter 5 in the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param cmp_par_5_used used compression parameter 5 + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_cmp_par5(struct cmp_entity *ent, uint32_t cmp_par_5_used) +{ + if (!ent) + return -1; + + if (cmp_par_5_used > UINT16_MAX) + return -1; + + ent->non_ima.cmp_par_5_used = cpu_to_be16(cmp_par_5_used); + + return 0; +} + + +/* + * @brief set the used spillover threshold 6 parameter in the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * @param spill_6_used used spillover threshold parameter 6 for non-imagette + * data_types + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_spill6(struct cmp_entity *ent, uint32_t spill_6_used) +{ + if (!ent) + return -1; + + if (spill_6_used > 0xFFFFFFUL) + return -1; + +#ifdef __LITTLE_ENDIAN + ent->non_ima.spill_6_used = cpu_to_be32(spill_6_used) >> 8; +#else + ent->non_ima.spill_6_used = spill_6_used; +#endif /* __LITTLE_ENDIAN */ + + return 0; +} + + +/** + * @brief set used compression parameter 6 in the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * @param cmp_par_6_used used compression parameter 6 + * + * @returns 0 on success, otherwise error + */ + +int cmp_ent_set_non_ima_cmp_par6(struct cmp_entity *ent, uint32_t cmp_par_6_used) +{ + if (!ent) + return -1; + + if (cmp_par_6_used > UINT16_MAX) + return -1; + + ent->non_ima.cmp_par_6_used = cpu_to_be16(cmp_par_6_used); + + return 0; +} + + +/** + * @brief get the ASW version identifier form the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the ASW version identifier on success, 0 on error + */ + +uint16_t cmp_ent_get_asw_version_id(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->asw_version_id); +} + + +/** + * @brief get the size of the compression entity form the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the size of the compression entity in bytes on success, 0 on error + */ + +uint32_t cmp_ent_get_size(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be32_to_cpu(ent->cmp_ent_size) >> 8; +#else + return ent->cmp_ent_size; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the original data size form the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the original size of the compressed data in bytes on success, 0 on error + */ + +uint32_t cmp_ent_get_original_size(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be32_to_cpu(ent->original_size) >> 8; +#else + return ent->original_size; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the compression start timestamp form the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the compression start timestamp on success, 0 on error + */ + +uint64_t cmp_ent_get_start_timestamp(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be64_to_cpu(ent->start_timestamp) >> 16; +#else + return ent->start_timestamp; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the coarse time form the compression start timestamp in the + * compression entity header + * + * @returns the coarse part of the compression start timestamp on success, 0 on + * error + */ + +uint32_t cmp_ent_get_coarse_start_time(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be32_to_cpu(ent->start_time.coarse); +} + + +/** + * @brief get the fine time form the compression start timestamp in the + * compression entity header + * + * @returns the fine part of the compression start timestamp on success, 0 on + * error + */ + +uint16_t cmp_ent_get_fine_start_time(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->start_time.fine); +} + + +/** + * @brief get the compression end timestamp form the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the compression end timestamp on success, 0 on error + */ + +uint64_t cmp_ent_get_end_timestamp(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be64_to_cpu(ent->end_timestamp << 16); +#else + return ent->end_timestamp; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the coarse time form the compression end timestamp in the + * compression entity header + * + * @returns the coarse part of the compression end timestamp on success, 0 on + * error + */ + +uint32_t cmp_ent_get_coarse_end_time(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be32_to_cpu(ent->end_time.coarse); +} + + +/** + * @brief get the fine time form the compression end timestamp in the + * compression entity header + * + * @returns the fine part of the compression end timestamp on success, 0 on + * error + */ + +uint16_t cmp_ent_get_fine_end_time(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->end_time.fine); +} + + +/** + * @brief get data_type form the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the data_type on success, DATA_TYPE_UNKOWN on error + */ + +enum cmp_ent_data_type cmp_ent_get_data_type(struct cmp_entity *ent) +{ + if (!ent) + return DATA_TYPE_UNKOWN; + + return be16_to_cpu(ent->data_type) & 0X7FFF; +} + + +/** + * @brief get raw bit form the data_type field of the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the data_type raw bit on success, 0 on error + */ + +int cmp_ent_get_data_type_raw_bit(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return (be16_to_cpu(ent->data_type) >> RAW_BIT_IN_DATA_TYPE) & 1U; +} + + +/** + * @brief get the used compression mode parameter from the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used compression on success, 0 on error + */ + +uint8_t cmp_ent_get_cmp_mode(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return ent->cmp_mode_used; +} + + +/** + * @brief get used model value from the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used model value used on success, 0 on error + */ + +uint8_t cmp_ent_get_model_value_used(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return ent->model_value_used; + +} + + +/** + * @brief get model id form the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the model identifier on success, 0 on error + */ + +uint16_t cmp_ent_get_model_id(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->model_id); +} + + +/** + * @brief get the model counter from the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the model counter on success, 0 on error + */ + +uint8_t cmp_ent_get_model_counter(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return ent->model_counter; +} + + +/** + * @brief get the used lossy compression parameter form the compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used lossy compression parameter on success, 0 on error + */ + +uint16_t cmp_ent_get_lossy_cmp_par(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->lossy_cmp_par_used); +} + + +/** + * @brief get the used spillover threshold parameter form the (adaptive) + * imagette specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used spillover threshold on success, 0 on error + */ + +uint16_t cmp_ent_get_ima_spill(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->ima.spill_used); +} + + +/** + * @brief get the used Golomb parameter form the (adaptive) imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used Golomb parameter on success, 0 on error + */ + +uint8_t cmp_ent_get_ima_golomb_par(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return ent->ima.golomb_par_used; +} + + +/** + * @brief get the used adaptive 1 spillover threshold parameter form the + * adaptive imagette specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used adaptive 1 spillover threshold on success, 0 on error + */ + +uint16_t cmp_ent_get_ima_ap1_spill(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->ima.ap1_spill_used); +} + + +/** + * @brief get the used adaptive 1 Golomb parameter form adaptive imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used adaptive 1 Golomb parameter on success, 0 on error + */ + +uint8_t cmp_ent_get_ima_ap1_golomb_par(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return ent->ima.ap1_golomb_par_used; +} + + +/** + * @brief get the used adaptive 2 spillover threshold parameter form the + * adaptive imagette specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used adaptive 2 spillover threshold on success, 0 on error + */ + +uint16_t cmp_ent_get_ima_ap2_spill(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->ima.ap2_spill_used); +} + + +/** + * @brief get the used adaptive 2 spillover threshold parameter form the + * adaptive imagette specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used adaptive 2 Golomb parameter on success, 0 on error + */ + +uint8_t cmp_ent_get_ima_ap2_golomb_par(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return ent->ima.ap2_golomb_par_used; +} + + +/** + * @brief get the used spillover threshold 1 parameter form the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used spillover threshold 1 parameter on success, 0 on error + */ + +uint32_t cmp_ent_get_non_ima_spill1(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be32_to_cpu(ent->non_ima.spill_1_used) >> 8; +#else + return ent->non_ima.spill_1_used; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the used compression parameter 1 form the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used compression parameter 1 on success, 0 on error + */ + +uint16_t cmp_ent_get_non_ima_cmp_par1(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->non_ima.cmp_par_1_used); +} + + +/** + * @brief get the used spillover threshold 2 parameter form the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used spillover threshold 2 parameter on success, 0 on error + */ + +uint32_t cmp_ent_get_non_ima_spill2(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be32_to_cpu(ent->non_ima.spill_2_used) >> 8; +#else + return ent->non_ima.spill_2_used; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the used compression parameter 2 form the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used compression parameter 2 on success, 0 on error + */ + +uint16_t cmp_ent_get_non_ima_cmp_par2(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->non_ima.cmp_par_2_used); +} + + +/** + * @brief get the used spillover threshold 3 parameter form the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used spillover threshold 3 parameter on success, 0 on error + */ + +uint32_t cmp_ent_get_non_ima_spill3(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be32_to_cpu(ent->non_ima.spill_3_used) >> 8; +#else + return ent->non_ima.spill_3_used; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the used compression parameter 3 form the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used compression parameter 3 on success, 0 on error + */ + +uint16_t cmp_ent_get_non_ima_cmp_par3(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->non_ima.cmp_par_3_used); +} + + +/** + * @brief get the used spillover threshold 4 parameter form the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used spillover threshold 4 parameter on success, 0 on error + */ + +uint32_t cmp_ent_get_non_ima_spill4(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be32_to_cpu(ent->non_ima.spill_4_used) >> 8; +#else + return ent->non_ima.spill_4_used; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the used compression parameter 4 form the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used compression parameter 4 on success, 0 on error + */ + +uint16_t cmp_ent_get_non_ima_cmp_par4(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->non_ima.cmp_par_4_used); +} + + +/** + * @brief get the used spillover threshold 5 parameter form the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used spillover threshold 5 parameter on success, 0 on error + */ + +uint32_t cmp_ent_get_non_ima_spill5(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be32_to_cpu(ent->non_ima.spill_5_used) >> 8; +#else + return ent->non_ima.spill_5_used; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the used compression parameter 5 form the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used compression parameter 5 on success, 0 on error + */ + +uint16_t cmp_ent_get_non_ima_cmp_par5(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->non_ima.cmp_par_5_used); +} + + +/** + * @brief get the used spillover threshold 6 parameter form the non-imagette + * specific compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used spillover threshold 6 parameter on success, 0 on error + */ + +uint32_t cmp_ent_get_non_ima_spill6(struct cmp_entity *ent) +{ + if (!ent) + return 0; + +#ifdef __LITTLE_ENDIAN + return be32_to_cpu(ent->non_ima.spill_6_used) >> 8; +#else + return ent->non_ima.spill_6_used; +#endif /* __LITTLE_ENDIAN */ +} + + +/** + * @brief get the used compression parameter 6 form the non-imagette specific + * compression entity header + * + * @param ent pointer to a compression entity + * + * @returns the used compression parameter 6 on success, 0 on error + */ + +uint16_t cmp_ent_get_non_ima_cmp_par6(struct cmp_entity *ent) +{ + if (!ent) + return 0; + + return be16_to_cpu(ent->non_ima.cmp_par_6_used); +} + + +/** + * @brief get the start address of the compressed data in the compression + * entity + * + * @param ent pointer to a compression entity + * + * @note this only works if the data_type in the compression entity is set + * + * @returns a pointer to buffer where the compressed data are located in entity + * on success, NULL on error + */ + +void *cmp_ent_get_data_buf(struct cmp_entity *ent) +{ + enum cmp_ent_data_type data_type; + + if (!ent) + return NULL; + + data_type = cmp_ent_get_data_type(ent); + + switch (data_type) { + case DATA_TYPE_IMAGETTE: + case DATA_TYPE_F_CAM_IMAGETTE: + return ent->ima.ima_cmp_dat; + case DATA_TYPE_IMAGETTE_ADAPTIVE: + case DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE: + return ent->ima.ap_ima_cmp_data; + case DATA_TYPE_SAT_IMAGETTE: + case DATA_TYPE_SAT_IMAGETTE_ADAPTIVE: + case DATA_TYPE_OFFSET: + case DATA_TYPE_BACKGROUND: + case DATA_TYPE_SMEARING: + case DATA_TYPE_S_FX: + case DATA_TYPE_S_FX_DFX: + case DATA_TYPE_S_FX_NCOB: + case DATA_TYPE_S_FX_DFX_NCOB_ECOB: + case DATA_TYPE_L_FX: + case DATA_TYPE_L_FX_DFX: + case DATA_TYPE_L_FX_NCOB: + case DATA_TYPE_L_FX_DFX_NCOB_ECOB: + case DATA_TYPE_F_FX: + case DATA_TYPE_F_FX_DFX: + case DATA_TYPE_F_FX_NCOB: + case DATA_TYPE_F_FX_DFX_NCOB_ECOB: + case DATA_TYPE_F_CAM_OFFSET: + case DATA_TYPE_F_CAM_BACKGROUND: + return ent->non_ima.cmp_data; + case DATA_TYPE_UNKOWN: + return NULL; + } + + return NULL; +} + + +/** + * @brief get the size of the compression entity header based on the set data + * product type in compression entity + * + * @param ent pointer to a compression entity + * + * @returns the size of the entity header in bytes on success, 0 on error + */ + +static uint32_t cmp_ent_get_hdr_size(struct cmp_entity *ent) +{ + return cmp_ent_cal_hdr_size(cmp_ent_get_data_type(ent)); +} + + +/** + * @brief get the size of the compressed data based on the set data product + * type and compressed entity size in the compression entity + * + * @param ent pointer to a compression entity + * + * @returns the size of the compressed data in bytes on success, 0 on error + */ + +uint32_t cmp_ent_get_cmp_data_size(struct cmp_entity *ent) +{ + uint32_t cmp_ent_size, header_size; + + header_size = cmp_ent_get_hdr_size(ent); + cmp_ent_size = cmp_ent_get_size(ent); + + if (header_size > cmp_ent_size) + return 0; + + return cmp_ent_size - header_size; +} + + +/** + * @brief set the parameters in the non-imagette specific compression + * entity header + * + * @param ent pointer to a compression entity + * @param info decompression information structure + * + * @returns 0 on success, otherwise error + * + * @warning this functions is not implemented jet + */ + +static int cmp_ent_write_non_ima_parameters(struct cmp_entity *ent, struct cmp_info *info) +{ + if (!info) + return -1; + (void)ent; + + printf("not implemented jet!\n"); + return -1; +#if 0 + if (cmp_ent_set_non_ima_spill1(ent, info->)) + return -1; + if (cmp_ent_set_non_ima_cmp_par1(ent, info->)) + return -1; + + if (cmp_ent_set_non_ima_spill2(ent, info->)) + return -1; + if (cmp_ent_set_non_ima_cmp_par2(ent, info->)) + return -1; + + if (cmp_ent_set_non_ima_spill3(ent, info->)) + return -1; + if (cmp_ent_set_non_ima_cmp_par3(ent, info->)) + return -1; + + if (cmp_ent_set_non_ima_spill4(ent, info->)) + return -1; + if (cmp_ent_set_non_ima_cmp_par4(ent, info->)) + return -1; + + if (cmp_ent_set_non_ima_spill5(ent, info->)) + return -1; + if (cmp_ent_set_non_ima_cmp_par5(ent, info->)) + return -1; + + if (cmp_ent_set_non_ima_spill6(ent, info->)) + return -1; + if (cmp_ent_set_non_ima_cmp_par6(ent, info->)) + return -1; + return 0; +#endif +} + + +/** + * @brief write the compression parameters in the entity header based on the data + * product type set in the entity and the provided decompression information + * struct + * + * @param ent pointer to a compression entity + * @param info decompression information structure + * @param cfg compression configuration structure for adaptive compression + * parameters (can be NULL) + * + * @returns 0 on success, negative on error + */ + +static int cmp_ent_write_cmp_pars(struct cmp_entity *ent, struct cmp_info *info, + struct cmp_cfg *cfg) +{ + uint32_t ent_cmp_data_size; + + if (!info) + return -1; + + ent_cmp_data_size = cmp_ent_get_cmp_data_size(ent); + + /* check if the entity can hold the compressed data */ + if (ent_cmp_data_size < cmp_bit_to_4byte(info->cmp_size)) + return -2; + + /* set compression parameter fields in the generic entity header */ + if (cmp_ent_set_original_size(ent, cmp_cal_size_of_data(info->samples_used, + info->cmp_mode_used))) + return -1; + if (cmp_ent_set_cmp_mode(ent, info->cmp_mode_used)) + return -1; + if (cmp_ent_set_data_type_raw_bit(ent, raw_mode_is_used(info->cmp_mode_used))) + return -1; + if (cmp_ent_set_model_value(ent, info->model_value_used)) + return -1; + if (cmp_ent_set_lossy_cmp_par(ent, info->round_used)) + return -1; + + switch (cmp_ent_get_data_type(ent)) { + case DATA_TYPE_IMAGETTE_ADAPTIVE: + case DATA_TYPE_SAT_IMAGETTE_ADAPTIVE: + case DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE: + if (!cfg) + return -1; + if (cmp_ent_set_ima_ap1_spill(ent, cfg->ap1_spill)) + return -1; + if (cmp_ent_set_ima_ap1_golomb_par(ent, cfg->ap1_golomb_par)) + return -1; + if (cmp_ent_set_ima_ap2_spill(ent, cfg->ap2_spill)) + return -1; + if (cmp_ent_set_ima_ap2_golomb_par(ent, cfg->ap2_golomb_par)) + return -1; + /* fall through */ + case DATA_TYPE_IMAGETTE: + case DATA_TYPE_SAT_IMAGETTE: + case DATA_TYPE_F_CAM_IMAGETTE: + if (cmp_ent_set_ima_spill(ent, info->spill_used)) + return -1; + if (cmp_ent_set_ima_golomb_par(ent, info->golomb_par_used)) + return -1; + break; + case DATA_TYPE_OFFSET: + case DATA_TYPE_BACKGROUND: + case DATA_TYPE_SMEARING: + case DATA_TYPE_S_FX: + case DATA_TYPE_S_FX_DFX: + case DATA_TYPE_S_FX_NCOB: + case DATA_TYPE_S_FX_DFX_NCOB_ECOB: + case DATA_TYPE_L_FX: + case DATA_TYPE_L_FX_DFX: + case DATA_TYPE_L_FX_NCOB: + case DATA_TYPE_L_FX_DFX_NCOB_ECOB: + case DATA_TYPE_F_FX: + case DATA_TYPE_F_FX_DFX: + case DATA_TYPE_F_FX_NCOB: + case DATA_TYPE_F_FX_DFX_NCOB_ECOB: + case DATA_TYPE_F_CAM_OFFSET: + case DATA_TYPE_F_CAM_BACKGROUND: + if (cmp_ent_write_non_ima_parameters(ent, info)) + return -1; + break; + case DATA_TYPE_UNKOWN: /* fall through */ + default: + return -1; + } + + return 0; +} + + +/** + * @brief create a compression entity by setting the size of the + * compression entity and the data product type in the entity header + * + * @param ent pointer to a compression entity; if NULL, the function + * returns the needed size + * @param data_type compression entity data product type + * @param cmp_size_byte size of the compressed data in bytes + * + * @note if the entity size is smaller than the largest header, the function + * rounds up the entity size to the largest header + * + * @returns the size of the compression entity or 0 on error + */ + +size_t cmp_ent_create(struct cmp_entity *ent, enum cmp_ent_data_type data_type, + uint32_t cmp_size_byte) +{ + int err; + uint32_t hdr_size = cmp_ent_cal_hdr_size(data_type); + size_t ent_size = hdr_size + cmp_size_byte; + uint32_t ent_size_cpy = ent_size; + + if (!hdr_size) + return 0; + + if (cmp_size_byte > CMP_ENTITY_MAX_SIZE) + return 0; + + if (ent_size < sizeof(struct cmp_entity)) + ent_size = sizeof(struct cmp_entity); + + if (!ent) + return ent_size; + + memset(ent, 0, hdr_size); + + err = cmp_ent_set_size(ent, ent_size_cpy); + if (err) + return 0; + + err = cmp_ent_set_data_type(ent, data_type); + if (err) + return 0; + + return ent_size; +} + + +/** + * @brief create a compression entity and set the header fields + * + * @note this function simplifies the entity setup by creating a entity and + * setting the header fields in one function call + * + * @param ent pointer to a compression entity; if NULL, the + * function returns the needed size + * @param data_type compression entity data product type + * @param asw_version_id applications software version identifier + * @param start_time compression start timestamp (coarse and fine) + * @param end_time compression end timestamp (coarse and fine) + * @param model_id model identifier + * @param model_counter model counter + * @param info decompression information structure + * @param cfg compression configuration structure for adaptive + * compression parameters (can be NULL) + * + * @returns the size of the compression entity or 0 on error + */ + +size_t cmp_ent_build(struct cmp_entity *ent, enum cmp_ent_data_type data_type, + uint16_t asw_version_id, uint64_t start_time, + uint64_t end_time, uint16_t model_id, uint8_t model_counter, + struct cmp_info *info, struct cmp_cfg *cfg) +{ + uint32_t ent_size; + + if (!info) + return 0; + + ent_size = cmp_ent_create(ent, data_type, cmp_bit_to_4byte(info->cmp_size)); + if (!ent_size) + return 0; + + if (ent) { + if (cmp_ent_set_asw_version_id(ent, asw_version_id)) + return 0; + if (cmp_ent_set_start_timestamp(ent, start_time)) + return 0; + if (cmp_ent_set_end_timestamp(ent, end_time)) + return 0; + if (cmp_ent_set_model_id(ent, model_id)) + return 0; + if (cmp_ent_set_model_counter(ent, model_counter)) + return 0; + if (cmp_ent_write_cmp_pars(ent, info, cfg)) + return 0; + } + + return ent_size; +} + + +/** + * @brief read in read in a imagette compression entity header to a info struct + * + * @param ent pointer to a compression entity + * @param info pointer to decompression information structure + * to store the read values + * + * @returns 0 on success; otherwise error + */ + +int cmp_ent_read_imagette_header(struct cmp_entity *ent, struct cmp_info *info) +{ + uint32_t original_size; + uint32_t sample_size; + + if (!ent) + return -1; + + if (!info) + return -1; + + info->cmp_mode_used = cmp_ent_get_cmp_mode(ent); + info->model_value_used = cmp_ent_get_model_value_used(ent); + info->round_used = cmp_ent_get_lossy_cmp_par(ent); + info->spill_used = cmp_ent_get_ima_spill(ent); + info->golomb_par_used = cmp_ent_get_ima_golomb_par(ent); + info->cmp_size = cmp_ent_get_cmp_data_size(ent)*CHAR_BIT; + + sample_size = size_of_a_sample(info->cmp_mode_used); + if (!sample_size) { + info->samples_used = 0; + return -1; + } + + original_size = cmp_ent_get_original_size(ent); + if (original_size % sample_size != 0) { + fprintf(stderr, "Error: original_size and cmp_mode compression header field are not compatible.\n"); + info->samples_used = 0; + return -1; + } + + info->samples_used = original_size / sample_size; + + if (cmp_ent_get_data_type_raw_bit(ent) != raw_mode_is_used(info->cmp_mode_used)) { + fprintf(stderr, "Error: The raw bit is set in Data Product Type Filed, but no raw compression mode is used.\n"); + return -1; + } + return 0; +} + + +#if 0 +int cmp_ent_read_adaptive_imagette_header(struct cmp_entity *ent, struct cmp_info *info) +{ + if (!ent) + return -1; + + if (!info) + return -1; + + if (cmp_ent_get_imagette_header(ent, info)) + return -1; + /* info->ap1_cmp_size_byte = cmp_ent_get_ap1_cmp_size_byte(ent); */ + /* info->ap2_cmp_size_byte = cmp_ent_get_ap2_cmp_size_byte(ent); */ + /* get ap1_spill and ap1/2 gpar*/ + + return 0; +} +#endif + + +#if __has_include(<time.h>) +/* + * @brief Covert a calendar time expressed as a struct tm object to time since + * epoch as a time_t object. The function interprets the input structure + * as representing Universal Coordinated Time (UTC). + * @note timegm is a GNU C Library extensions, not standardized. This function + * is used as a portable alternative + * @note The function is thread-unsafe + * + * @param tm pointer to a tm object specifying local calendar time to convert + * + * @returns time since epoch as a time_t object on success or -1 if time cannot be represented as a time_t object + * + * @see http://www.catb.org/esr/time-programming/#_unix_time_and_utc_gmt_zulu + */ + +static time_t my_timegm(struct tm *tm) +{ +#if defined(_WIN32) || defined(_WIN64) + return _mkgmtime(tm); +#else + time_t ret; + char *tz; + + tz = getenv("TZ"); + if (tz) + tz = strdup(tz); + setenv("TZ", "", 1); + tzset(); + ret = mktime(tm); + if (tz) { + setenv("TZ", tz, 1); + free(tz); + } else + unsetenv("TZ"); + tzset(); + return ret; +#endif +} + + +/* + * @brief Generate a timestamp for the compression header + * + * @param ts pointer to object of type struct timespec of the timestamp time, null for now + * + * @returns returns compression header timestamp or 0 on error + */ + +uint64_t cmp_ent_create_timestamp(const struct timespec *ts) +{ + struct tm epoch_date = EPOCH_DATE; + struct timespec epoch = { my_timegm(&epoch_date), 0 }; + struct timespec now = { 0, 0 }; + double seconds; + uint64_t coarse, fine; + + if (epoch.tv_sec == -1) + return 0; + + if (ts) + now = *ts; + else + clock_gettime(CLOCK_REALTIME, &now); + + seconds = ((double)now.tv_sec + 1.0e-9 * now.tv_nsec) - + ((double)epoch.tv_sec + 1.0e-9 * epoch.tv_nsec); + + coarse = (uint64_t)seconds; + fine = (uint64_t)((seconds - coarse) * 256 * 256); + + return coarse * 256 * 256 + fine; +} +#endif + + +/** + * @brief print the content of the compression entity header + * + * @param ent pointer to a compression entity + */ + +void cmp_ent_print_header(struct cmp_entity *ent) +{ + uint8_t *p = (uint8_t *)ent; + uint32_t hdr_size = cmp_ent_get_hdr_size(ent); + size_t i; + + for (i = 0; i < hdr_size; ++i) { + printf("%02X ", p[i]); + if (i && !((i+1) % 32)) + printf("\n"); + } + printf("\n"); +} + + +/** + * @brief print the compressed data of the entity + * + * @param ent pointer to a compression entity + */ + +void cmp_ent_print_data(struct cmp_entity *ent) +{ + uint8_t *p = cmp_ent_get_data_buf(ent); + size_t data_size = cmp_ent_get_cmp_data_size(ent); + size_t i; + + if (!p) + return; + + for (i = 0; i < data_size; ++i) { + printf("%02X ", p[i]); + if (i && !((i+1) % 32)) + printf("\n"); + } + printf("\n"); +} + + +/** + * @brief print the entire compressed entity header plus data + * + * @param ent pointer to a compression entity + */ + +void cmp_ent_print(struct cmp_entity *ent) +{ + printf("compression entity header:\n"); + cmp_ent_print_header(ent); + printf("compressed data in the compressed entity:\n"); + cmp_ent_print_data(ent); +} + + +/** + * @brief parse the generic compressed entity header + * + * @param ent pointer to a compression entity + */ + +static void cmp_ent_parse_generic_header(struct cmp_entity *ent) +{ + uint32_t asw_version_id, cmp_ent_size, original_size, cmp_mode_used, + model_value_used, model_id, model_counter, lossy_cmp_par_used, + start_coarse_time, end_coarse_time; + uint16_t start_fine_time, end_fine_time; + enum cmp_ent_data_type data_type; + int raw_bit; + + asw_version_id = cmp_ent_get_asw_version_id(ent); + if (asw_version_id & CMP_TOOL_VERSION_ID_BIT) { + uint16_t major = (asw_version_id & 0x7F00U) >> 8U; + uint16_t minor = asw_version_id & 0x00FFU; + printf("Compressed with cmp_tool version: %u.%02u\n", major,minor); + } else + printf("ICU ASW Version ID: %u\n", asw_version_id); + + cmp_ent_size = cmp_ent_get_size(ent); + printf("Compression Entity Size: %u byte\n", cmp_ent_size); + + original_size = cmp_ent_get_original_size(ent); + printf("Original Data Size: %u byte\n", original_size); + + start_coarse_time = cmp_ent_get_coarse_start_time(ent); + printf("Compression Coarse Start Time: %u\n", start_coarse_time); + + start_fine_time = cmp_ent_get_fine_start_time(ent); + printf("Compression Fine Start Time: %d\n", start_fine_time); + + end_coarse_time = cmp_ent_get_coarse_end_time(ent); + printf("Compression Coarse End Time: %u\n", end_coarse_time); + + end_fine_time = cmp_ent_get_fine_end_time(ent); + printf("Compression Fine End Time: %d\n", end_fine_time); + +#if __has_include(<time.h>) + { + struct tm epoch_date = EPOCH_DATE; + time_t time = my_timegm(&epoch_date) + start_coarse_time; + printf("Data were compressed on (local time): %s", ctime(&time)); + } +#endif + printf("The compression took %f second\n", end_coarse_time - start_coarse_time + + ((end_fine_time - start_fine_time)/256./256.)); + + data_type = cmp_ent_get_data_type(ent); + if (data_type != DATA_TYPE_UNKOWN) + printf("Data Product Type: %d\n", data_type); + else + printf("Data Product Type: unknown!"); + + raw_bit = cmp_ent_get_data_type_raw_bit(ent); + printf("RAW bit in the Data Product Type is%s set", raw_bit ? "" : "not"); + + cmp_mode_used = cmp_ent_get_cmp_mode(ent); + printf("Used Compression Mode: %u\n", cmp_mode_used); + + model_value_used = cmp_ent_get_model_value_used(ent); + printf("Used Model Updating Weighing Value: %u\n", model_value_used); + + model_id = cmp_ent_get_model_id(ent); + printf("Model ID: %u\n", model_id); + + model_counter = cmp_ent_get_model_counter(ent); + printf("Model Counter: %u\n", model_counter); + + lossy_cmp_par_used = cmp_ent_get_lossy_cmp_par(ent); + printf("Used Lossy Compression Parameters: %u\n", lossy_cmp_par_used); +} + + +/** + * @brief parse the imagette specific compressed entity header + * + * @param ent pointer to a compression entity + */ + +static void cmp_ent_parese_imagette_header(struct cmp_entity *ent) +{ + uint32_t spill_used, golomb_par_used; + + spill_used = cmp_ent_get_ima_spill(ent); + printf("Used Spillover Threshold Parameter: %u\n", spill_used); + + golomb_par_used = cmp_ent_get_ima_golomb_par(ent); + printf("Used Golomb Parameter: %u\n", golomb_par_used); +} + + +/** + * @brief parse the adaptive imagette specific compressed entity header + * + * @param ent pointer to a compression entity + */ + +static void cmp_ent_parese_adaptive_imagette_header(struct cmp_entity *ent) +{ + uint32_t spill_used, golomb_par_used, ap1_spill_used, + ap1_golomb_par_used, ap2_spill_used, ap2_golomb_par_used; + + spill_used = cmp_ent_get_ima_spill(ent); + printf("Used Spillover Threshold Parameter: %u\n", spill_used); + + golomb_par_used = cmp_ent_get_ima_golomb_par(ent); + printf("Used Golomb Parameter: %u\n", golomb_par_used); + + ap1_spill_used = cmp_ent_get_ima_ap1_spill(ent); + printf("Used Adaptive 1 Spillover Threshold Parameter: %u\n", ap1_spill_used); + + ap1_golomb_par_used = cmp_ent_get_ima_ap1_golomb_par(ent); + printf("Used Adaptive 1 Golomb Parameter: %u\n", ap1_golomb_par_used); + + ap2_spill_used = cmp_ent_get_ima_ap2_spill(ent); + printf("Used Adaptive 2 Spillover Threshold Parameter: %u\n", ap2_spill_used); + + ap2_golomb_par_used = cmp_ent_get_ima_ap2_golomb_par(ent); + printf("Used Adaptive 2 Golomb Parameter: %u\n", ap2_golomb_par_used); +} + + +/** + * @brief parse the specific compressed entity header + * + * @param ent pointer to a compression entity + */ + +static void cmp_ent_parese_specific_header(struct cmp_entity *ent) +{ + enum cmp_ent_data_type data_type = cmp_ent_get_data_type(ent); + + switch (data_type) { + case DATA_TYPE_IMAGETTE: + cmp_ent_parese_imagette_header(ent); + break; + case DATA_TYPE_IMAGETTE_ADAPTIVE: + cmp_ent_parese_adaptive_imagette_header(ent); + break; + default: + printf("Data Product Type not supported!\n"); + break; + } +} + + +/** + * @brief parse the compressed entity header + * + * @param ent pointer to a compression entity + */ + +void cmp_ent_parse(struct cmp_entity *ent) +{ + cmp_ent_parse_generic_header(ent); + + cmp_ent_parese_specific_header(ent); +} diff --git a/lib/cmp_icu.c b/lib/cmp_icu.c index 21aabe7d9e444ca494a65f4720a1b2c59f4159f2..2a7c6a188b9fb40be61831824a62254e3feb907b 100644 --- a/lib/cmp_icu.c +++ b/lib/cmp_icu.c @@ -1128,8 +1128,8 @@ static unsigned int put_n_bits32(unsigned int value, unsigned int bitOffset, /* check if destination buffer is large enough */ /* TODO: adapt that to the other science products */ - if ((bitOffset + nBits) > (((dest_len+1)/2)*2 * 16)) { - debug_print("Error: The icu_output_buf buffer is not small to hold the compressed data.\n"); + if ((bitOffset + nBits) > ((dest_len&~0x1U)*16)) { + debug_print("Error: The buffer for the compressed data is too small to hold the compressed data. Try a larger buffer_length parameter.\n"); return -2U; } @@ -1257,21 +1257,19 @@ static int encode_raw(struct cmp_cfg *cfg, struct encoder_struct *enc) static int encode_raw_16(struct cmp_cfg *cfg, struct encoder_struct *enc) { int err; + size_t i; - err = encode_raw(cfg, enc); - if (err) - return err; - -#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) - { - size_t i; - uint16_t *p = cfg->icu_output_buf; - - for (i = 0; i < cfg->samples; i++) - cpu_to_be16s(&p[i]); + enc->cmp_size = 0; + for (i = 0; i < cfg->samples; i++) { + uint16_t *p = cfg->input_buf; + err = put_n_bits32(p[i], enc->cmp_size, 16, cfg->icu_output_buf, + cfg->buffer_length); + if (err <= 0) + return err; + enc->cmp_size += 16; } -#endif /* __BYTE_ORDER__ */ + return 0; } @@ -1330,7 +1328,9 @@ static int encode_outlier_zero(uint32_t value_to_encode, int max_bits, return -1; /* use zero as escape symbol */ - encode_normal(0, cfg, enc); + err = encode_normal(0, cfg, enc); + if (err) + return err; /* put the data unencoded in the bitstream */ err = put_n_bits32(value_to_encode, enc->cmp_size, max_bits, @@ -1407,7 +1407,9 @@ static int encode_outlier_multi(uint32_t value_to_encode, struct cmp_cfg *cfg, unencoded_data_len = (escape_sym_offset + 1) * 2; /* put the escape symbol in the bitstream */ - encode_normal(escape_sym, cfg, enc); + err = encode_normal(escape_sym, cfg, enc); + if (err) + return err; /* put the unencoded data in the bitstream */ err = put_n_bits32(unencoded_data, enc->cmp_size, unencoded_data_len, @@ -1606,6 +1608,7 @@ static int encode_S_FX_EFX_NCOB_ECOB(struct cmp_cfg *cfg, struct encoder_struct return 0; } + /* pad the bitstream with zeros */ int pad_bitstream(struct cmp_cfg *cfg, uint32_t cmp_size) { @@ -1615,7 +1618,7 @@ int pad_bitstream(struct cmp_cfg *cfg, uint32_t cmp_size) return -1; /* is padding needed */ - if (!raw_mode_is_used(cfg->cmp_mode) && cmp_size) { + if (cmp_size) { int n_pad_bits = 32U - (cmp_size & 0x1f); if (n_pad_bits < 32) { n_bits = put_n_bits32(0, cmp_size, n_pad_bits, @@ -1699,9 +1702,11 @@ uint32_t cmp_encode_data(struct cmp_cfg *cfg) { size_t i; uint32_t *p = (uint32_t *)cfg->icu_output_buf; - if (p) - for (i = 0; i < (enc.cmp_size + n_bits)/32; i++) + + if (p) { + for (i = 0; i < cmp_bit_to_4byte(enc.cmp_size)/sizeof(uint32_t); i++) cpu_to_be32s(&p[i]); + } } #endif /*__BYTE_ORDER__ */ diff --git a/lib/cmp_rdcu.c b/lib/cmp_rdcu.c index caf9c6ee2e50c658a4a9acad51f364d382901492..c24ebed0e8eefb9e8e297ed1919d0b72e851442a 100644 --- a/lib/cmp_rdcu.c +++ b/lib/cmp_rdcu.c @@ -665,7 +665,7 @@ int rdcu_read_cmp_bitstream(const struct cmp_info *info, void *output_buf) return -1; /* calculate the need bytes for the bitstream */ - s = size_of_bitstream(info->cmp_size); + s = cmp_bit_to_4byte(info->cmp_size); if (output_buf == NULL) return (int)s; @@ -700,7 +700,7 @@ int rdcu_read_model(const struct cmp_info *info, void *model_buf) return -1; /* calculate the need bytes for the model */ - s = size_of_model(info->samples_used, info->cmp_mode_used); + s = cmp_cal_size_of_data(info->samples_used, info->cmp_mode_used); if (model_buf == NULL) return (int)s; diff --git a/lib/cmp_support.c b/lib/cmp_support.c index c0783197adbfd4614e9c8cef6c2a898aa0abe1ec..a9459718e1a8c79cc913e574e1072b268fff0f72 100644 --- a/lib/cmp_support.c +++ b/lib/cmp_support.c @@ -561,27 +561,28 @@ size_t size_of_a_sample(unsigned int cmp_mode) /** * @brief calculate the need bytes to hold a bitstream * - * @param cmp_size compressed data size, measured in bits + * @param cmp_size_bit compressed data size, measured in bits * * @returns the size in bytes to store the hole bitstream + * @note we round up the result to multiples of 4 bytes */ -unsigned int size_of_bitstream(unsigned int cmp_size) +unsigned int cmp_bit_to_4byte(unsigned int cmp_size_bit) { - return (((cmp_size >> 3) + 3) & ~0x3U); + return (((cmp_size_bit + 7) / 8) + 3) & ~0x3UL; } /** - * @brief calculate the need bytes for the model + * @brief calculate the need bytes for the data * - * @param samples amount of model samples - * @param cmp_mode compression mode + * @param samples number of data samples + * @param cmp_mode used compression mode * - * @returns the size in bytes to store the hole bitstream + * @returns the size in bytes to store the data sample */ -unsigned int size_of_model(unsigned int samples, unsigned int cmp_mode) +unsigned int cmp_cal_size_of_data(unsigned int samples, unsigned int cmp_mode) { return samples * size_of_a_sample(cmp_mode); } diff --git a/lib/cmp_tool_lib.c b/lib/cmp_tool_lib.c index 41f7763f80ea0cff5e4dcb23325deac548820ccd..478faf18c0c21949579be724ce2b2340ac3aec18 100644 --- a/lib/cmp_tool_lib.c +++ b/lib/cmp_tool_lib.c @@ -39,12 +39,13 @@ void print_help(const char *program_name) printf("usage: %s [options] [<argument>]\n", program_name); printf("General Options:\n"); printf(" -h, --help Print this help text and exit\n"); - printf(" -V, --version Print program version and exit\n"); - printf(" -v, --verbose Print various debugging information\n"); + printf(" -o <prefix> Use the <prefix> for output files\n"); printf(" -n, --model_cfg Print a default model configuration and exit\n"); printf(" --diff_cfg Print a default 1d-differencing configuration and exit\n"); + printf(" --no_header Do not add a compression entity header in front of the compressed data\n"); printf(" -a, --rdcu_par Add additional RDCU control parameters\n"); - printf(" -o <prefix> Use the <prefix> for output files\n"); + printf(" -V, --version Print program version and exit\n"); + printf(" -v, --verbose Print various debugging information\n"); printf("Compression Options:\n"); printf(" -c <file> File containing the compressing configuration\n"); printf(" -d <file> File containing the data to be compressed\n"); @@ -52,9 +53,9 @@ void print_help(const char *program_name) printf(" --rdcu_pkt Generate RMAP packets for an RDCU compression\n"); printf(" --last_info <.info file> Generate RMAP packets for an RDCU compression with parallel read of the last results\n"); printf("Decompression Options:\n"); - printf(" -i <file> File containing the decompression information\n"); printf(" -d <file> File containing the compressed data\n"); printf(" -m <file> File containing the model of the compressed data\n"); + printf(" -i <file> File containing the decompression information (required if --no_header was used)\n"); printf("Guessing Options:\n"); printf(" --guess <mode> Search for a good configuration for compression <mode>\n"); printf(" -d <file> File containing the data to be compressed\n"); @@ -67,8 +68,8 @@ void print_help(const char *program_name) * @brief opens a file with a name that is a concatenation of directory and file * name. * - * @param dirname first sting of concatenation - * @param filename security sting of concatenation + * @param dirname first string of concatenation + * @param filename security string of concatenation * * @return a pointer to the opened file * @@ -113,7 +114,7 @@ static FILE *open_file(const char *dirname, const char *filename) /** - * @brief write a uint16_t buffer to a output file + * @brief write a uint16_t buffer to an output file * * @param buf the buffer to write a file * @param buf_len length of the buffer @@ -174,7 +175,7 @@ int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char /** - * @brief write a buffer to a output file + * @brief write a buffer to an output file * * @param buf the buffer to write a file * @param buf_size size of the buffer in bytes @@ -278,7 +279,7 @@ static void remove_comments(char *s) /** - * @brief convert RDCU SRAM Address sting to integer + * @brief convert RDCU SRAM Address string to integer * * @param addr string of the address * @@ -319,8 +320,8 @@ static int sram_addr_to_int(const char *addr) /** * @brief Interprets an uint32_t integer value in a byte string * - * @param dep_str description sting of the read in value - * @param val_str value sting contain the value to convert in uint32_t + * @param dep_str description string of the read in value + * @param val_str value string contain the value to convert in uint32_t * @param red_val address for storing the converted result * * @returns 0 on success, error otherwise @@ -328,9 +329,9 @@ static int sram_addr_to_int(const char *addr) * @see https://eternallyconfuzzled.com/atoi-is-evil-c-learn-why-atoi-is-awful */ -static int atoui32(const char *dep_str, const char *val_str, uint32_t *red_val) +int atoui32(const char *dep_str, const char *val_str, uint32_t *red_val) { - long temp; + unsigned long temp; char *end = NULL; if (!dep_str) @@ -343,21 +344,20 @@ static int atoui32(const char *dep_str, const char *val_str, uint32_t *red_val) return -1; errno = 0; - temp = strtol(val_str, &end, 10); - if (end != val_str && errno != ERANGE && temp >= 0 && - temp <= UINT32_MAX) { + temp = strtoul(val_str, &end, 0); + if (end != val_str && errno != ERANGE && temp <= UINT32_MAX) { *red_val = (uint32_t)temp; - return 0; } else { fprintf(stderr, "%s: Error read in %s.\n", PROGRAM_NAME, dep_str); *red_val = 0; return -1; } + return 0; } /** - * @brief parse a compression mode vale string to a integer + * @brief parse a compression mode vale string to an integer * @note string can be either a number or the name of the compression mode * * @param cmp_mode_str string containing the compression mode value to parse @@ -366,7 +366,7 @@ static int atoui32(const char *dep_str, const char *val_str, uint32_t *red_val) * @returns 0 on success, error otherwise */ -uint32_t cmp_mode_parse(const char *cmp_mode_str, uint32_t *cmp_mode) +int cmp_mode_parse(const char *cmp_mode_str, uint32_t *cmp_mode) { size_t j; static const struct { @@ -420,6 +420,8 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg) { char *token1, *token2; char line[MAX_CONFIG_LINE]; + enum {CMP_MODE, GOLOMB_PAR, SPILL, SAMPLES, BUFFER_LENGTH, LAST_ITEM}; + int j, must_read_items[LAST_ITEM] = {0}; if (!fp) return -1; @@ -449,6 +451,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg) if (!strcmp(token1, "cmp_mode")) { + must_read_items[CMP_MODE] = 1; if (isalpha(*token2)) { /* check if mode is given as text or val*/ if (!strcmp(token2, "MODE_RAW")) { cfg->cmp_mode = 0; @@ -472,6 +475,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg) } fprintf(stderr, "%s: Error read in cmp_mode.\n", PROGRAM_NAME); + return -1; } else { if (atoui32(token1, token2, &cfg->cmp_mode)) return -1; @@ -481,11 +485,13 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg) if (!strcmp(token1, "golomb_par")) { if (atoui32(token1, token2, &cfg->golomb_par)) return -1; + must_read_items[GOLOMB_PAR] = 1; continue; } if (!strcmp(token1, "spill")) { if (atoui32(token1, token2, &cfg->spill)) return -1; + must_read_items[SPILL] = 1; continue; } if (!strcmp(token1, "model_value")) { @@ -554,6 +560,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg) if (!strcmp(token1, "samples")) { if (atoui32(token1, token2, &cfg->samples)) return -1; + must_read_items[SAMPLES] = 1; continue; } if (!strcmp(token1, "rdcu_buffer_adr")) { @@ -570,9 +577,24 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg) if (!strcmp(token1, "buffer_length")) { if (atoui32(token1, token2, &cfg->buffer_length)) return -1; + must_read_items[BUFFER_LENGTH] = 1; continue; } } + + if (raw_mode_is_used(cfg->cmp_mode)) + if (must_read_items[CMP_MODE] == 1 && + must_read_items[BUFFER_LENGTH] == 1) + return 0; + + for (j = 0; j < LAST_ITEM; j++) { + if (must_read_items[j] < 1) { + fprintf(stderr, "%s: Some parameters are missing. Check if the following parameters: cmp_mode, golomb_par, spill, samples and buffer_length are all set in the configuration file.\n", + PROGRAM_NAME); + return -1; + } + } + return 0; } @@ -648,6 +670,9 @@ static int parse_info(FILE *fp, struct cmp_info *info) { char *token1, *token2; char line[MAX_CONFIG_LINE]; + enum {CMP_MODE_USED, GOLOMB_PAR_USED, SPILL_USED, SAMPLES_USED, + CMP_SIZE, LAST_ITEM}; + int j, must_read_items[LAST_ITEM] = {0}; if (!fp) return -1; @@ -678,6 +703,7 @@ static int parse_info(FILE *fp, struct cmp_info *info) if (!strcmp(token1, "cmp_mode_used")) { + must_read_items[CMP_MODE_USED] = 1; if (isalpha(*token2)) { /* check if mode is given as text or val*/ if (!strcmp(token2, "MODE_RAW")) { info->cmp_mode_used = 0; @@ -701,7 +727,7 @@ static int parse_info(FILE *fp, struct cmp_info *info) } fprintf(stderr, "%s: Error read in cmp_mode_used.\n", PROGRAM_NAME); - return -1; + return -1; } else { uint32_t tmp; @@ -745,6 +771,7 @@ static int parse_info(FILE *fp, struct cmp_info *info) if (atoui32(token1, token2, &tmp)) return -1; info->spill_used = (unsigned char)tmp; + must_read_items[SPILL_USED] = 1; continue; } if (!strcmp(token1, "golomb_par_used")) { @@ -753,16 +780,19 @@ static int parse_info(FILE *fp, struct cmp_info *info) if (atoui32(token1, token2, &tmp)) return -1; info->golomb_par_used = tmp; + must_read_items[GOLOMB_PAR_USED] = 1; continue; } if (!strcmp(token1, "samples_used")) { if (atoui32(token1, token2, &info->samples_used)) return -1; + must_read_items[SAMPLES_USED] = 1; continue; } if (!strcmp(token1, "cmp_size")) { if (atoui32(token1, token2, &info->cmp_size)) return -1; + must_read_items[CMP_SIZE] = 1; continue; } @@ -814,6 +844,21 @@ static int parse_info(FILE *fp, struct cmp_info *info) continue; } } + + if (raw_mode_is_used(info->cmp_mode_used)) + if (must_read_items[CMP_MODE_USED] == 1 && + must_read_items[SAMPLES_USED] == 1 && + must_read_items[CMP_SIZE] == 1) + return 0; + + for (j = 0; j < LAST_ITEM; j++) { + if (must_read_items[j] < 1) { + fprintf(stderr, "%s: Some parameters are missing. Check if the following parameters: cmp_mode_used, golomb_par_used, spill_used and samples_used are all set in the information file.\n", + PROGRAM_NAME); + return -1; + } + } + return 0; } @@ -831,6 +876,7 @@ static int parse_info(FILE *fp, struct cmp_info *info) int read_cmp_info(const char *file_name, struct cmp_info *info, int verbose_en) { + int err; FILE *fp; if (!file_name) @@ -850,9 +896,11 @@ int read_cmp_info(const char *file_name, struct cmp_info *info, int verbose_en) return -1; } - parse_info(fp, info); - + err = parse_info(fp, info); fclose(fp); + if (err) + return -1; + if (verbose_en) { printf("\n\n"); @@ -865,8 +913,8 @@ int read_cmp_info(const char *file_name, struct cmp_info *info, int verbose_en) /** - * @brief reads n_word words of a hex-encoded data (or model) file to - * a uint8_t buffer + * @brief reads n_word words of a hex-encoded data (or model) file to a uint8_t + * buffer * * @note data must be encoded as 2 hex digits separated by a white space or new * line character e.g. "AB CD 12\n34 AB 12\n" @@ -918,7 +966,7 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int ver if (!buf) /* finished counting the sample */ break; - fprintf(stderr, "%s: %s: Error: The files does not contain enough data as given by the n_word parameter.\n", + fprintf(stderr, "%s: %s: Error: The files do not contain enough data as requested.\n", PROGRAM_NAME, file_name); goto error; } @@ -986,6 +1034,8 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int ver fclose(fp); + if (n == 0) + fprintf(stderr, "%s: %s: Warning: The file is empty.\n", PROGRAM_NAME, file_name); if (verbose_en && buf) printf("\n\n"); @@ -1001,7 +1051,7 @@ error: * @brief reads the number of uint16_t samples of a hex-encoded data (or model) * file into a buffer * - * @note data must be encode has 2 hex digits separated by a white space or new + * @note data must be encoded as 2 hex digits separated by a white space or new * line character e.g. "AB CD 12 34 AB 12\n" * * @param file_name data/model file name @@ -1042,7 +1092,7 @@ ssize_t read_file16(const char *file_name, uint16_t *buf, uint32_t samples, * @brief reads the number of uint32_t samples of a hex-encoded data (or model) * file into a buffer * - * @note data must be encode has 2 hex digits separated by a white space or new + * @note data must be encoded as 2 hex digits separated by a white space or new * line character e.g. "AB CD 12 34 AB 12\n" * * @param file_name data/model file name @@ -1077,6 +1127,134 @@ ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t samples, return size; } +#if 0 +read_file_data(const char *file_name, cmp_mode, samples, verbose_en) + + cfg.samples = size/size_of_a_sample(cfg.cmp_mode); + printf("\nNo samples parameter set. Use samples = %u.\n... ", + cfg.samples); + } +#endif + +/** + * @brief reads a file containing a compression entity + * + * @note data must be encoded as 2 hex digits separated by a white space or new + * line character e.g. "AB CD 12 34 AB 12\n" + * + * @param file_name file name of the file containing the compression entity + * @param ent pointer to the buffer where the content of the file is written (can be NULL) + * @param ent_size size in bytes of the compression entity to read in + * @param verbose_en print verbose output if not zero + * + * @returns the size in bytes to store the file content; negative on error + */ + +ssize_t read_file_cmp_entity(const char *file_name, struct cmp_entity *ent, + uint32_t ent_size, int verbose_en) +{ + ssize_t size; + + size = read_file8(file_name, (uint8_t *)ent, ent_size, 0); + if (size < 0) + return size; + if (size < GENERIC_HEADER_SIZE) { + fprintf(stderr, "%s: %s: Error: The file is too small to contain a compression entity header.\n", + PROGRAM_NAME, file_name); + return -1; + } + + if (ent) { + size_t i; + uint32_t *cmp_data; + uint32_t cmp_data_len_32; + + if (size != cmp_ent_get_size(ent)) { + fprintf(stderr, "%s: %s: The size of the compression entity set in the header of the compression entity is not the same size as the read-in file has.\n", + PROGRAM_NAME, file_name); + return -1; + } + + if (verbose_en) + cmp_ent_parse(ent); + + if (cmp_ent_get_cmp_data_size(ent) & 0x3) { + fprintf(stderr, "%s: %s: Error: The compressed data are not correct formatted. Expected multiple of 4 hex words.\n", + PROGRAM_NAME, file_name); + return -1; + + } + + cmp_data = (uint32_t *)cmp_ent_get_data_buf(ent); + if (!cmp_data) { + fprintf(stderr, "%s: %s: Error: Compression data type is not supported.\n", + PROGRAM_NAME, file_name); + return -1; + } + cmp_data_len_32 = cmp_ent_get_cmp_data_size(ent)/sizeof(uint32_t); + for (i = 0; i < cmp_data_len_32; i++) + be32_to_cpus(&cmp_data[i]); + + if (verbose_en) { + printf("\ncompressed data:\n"); + for (i = 0; i < cmp_data_len_32; i++) { + printf("%08X ", cmp_data[i]); + if (i && !((i+1) % 4)) + printf("\n"); + } + printf("\n"); + } + } + return size; +} + + +/* + * @brief generate from the cmp_tool version string a version_id for the + * compression entity header + * + * @param version number string like: 1.04 + * + * @returns version_id for the compression header; 0 on error + */ + +uint16_t cmp_tool_gen_version_id(const char *version) +{ + char *copy, *token; + unsigned int n; + uint16_t version_id; + + /* + * version_id bits: msb |xxxx xxxx | xxxx xxxx| lsb + * ^^^ ^^^^ | ^^^^ ^^^^ + * mayor num| minor version number + * ^ + * CMP_TOOL_VERSION_ID_BIT + */ + copy = strdup(version); + token = strtok(copy, "."); + n = atoi(token); + if (n > UINT8_MAX) { + free(copy); + return 0; + } + version_id = ((uint16_t)n) << 8U; + if (version_id & CMP_TOOL_VERSION_ID_BIT) { + free(copy); + return 0; + } + + token = strtok(NULL, "."); + n = atoi(token); + free(copy); + if (n > UINT8_MAX) + return 0; + + version_id |= n; + + return version_id |= CMP_TOOL_VERSION_ID_BIT; +} + /** * @brief write compression configuration to a stream diff --git a/lib/decmp.c b/lib/decmp.c index e735c3db5fd85a9b054eb90ad48a5304e76023d2..4953f5d55566e47e1307cf423e626e5795850d48 100644 --- a/lib/decmp.c +++ b/lib/decmp.c @@ -765,8 +765,9 @@ static int de_map_to_pos(void *decompressed_data, const struct cmp_info *info) } -static unsigned int GetNBits32 (uint32_t *p_value, unsigned int bitOffset, - unsigned int nBits, const unsigned int *srcAddr) +static unsigned int get_n_bits32(uint32_t *p_value, unsigned int bitOffset, + unsigned int nBits, const unsigned int *srcAddr, + size_t src_len_bit) { const unsigned int *localAddr; unsigned int bitsLeft, bitsRight, localEndPos; @@ -780,9 +781,14 @@ static unsigned int GetNBits32 (uint32_t *p_value, unsigned int bitOffset, return 0; if (!p_value) return 0; + if ((bitOffset + nBits) > src_len_bit) { + debug_print("Error: Buffer overflow detected.\n"); + return 0; + } /* separate the bitOffset into word offset (set localAddr pointer) and * local bit offset (bitsLeft) */ + localAddr = srcAddr + (bitOffset >> 5); bitsLeft = bitOffset & 0x1f; @@ -948,18 +954,29 @@ static int decode_raw(const void *compressed_data, const struct cmp_info static int decode_raw_16(const void *compressed_data, const struct cmp_info *info, uint16_t *const decompressed_data) { - int err = decode_raw(compressed_data, info, decompressed_data); - if (err) - return err; + size_t i; + uint16_t *p = decompressed_data; + uint32_t read_pos = 0; + unsigned int read_bits; + uint32_t read_val; -#if defined(__LITTLE_ENDIAN) - { - size_t i; - for (i = 0; i < info->samples_used; i++) { - decompressed_data[i] = cpu_to_be16(decompressed_data[i]); - } + if (!info) + return -1; + if (info->samples_used == 0) + return 0; + if (!compressed_data) + return -1; + if (!decompressed_data) + return -1; + + for (i = 0; i < info->samples_used; ++i) { + read_bits = get_n_bits32(&read_val, read_pos, 16, + compressed_data, info->cmp_size); + if (!read_bits) + return -1; + read_pos += 16; + p[i] =read_val; } -#endif return 0; } @@ -1022,8 +1039,8 @@ static unsigned int decode_normal(const void *compressed_data, else n_read_bits = max_cw_len; - read_bits = GetNBits32(&read_val, read_pos, n_read_bits, - compressed_data); + read_bits = get_n_bits32(&read_val, read_pos, n_read_bits, + compressed_data, info->cmp_size); if (!read_bits) return -1U; @@ -1073,10 +1090,8 @@ static unsigned int decode_zero(const void *compressed_data, unsigned int n_bits; uint32_t unencoded_val; - if (read_pos + max_cw_len > info->cmp_size) /* check buffer overflow */ - return -1U; - n_bits = GetNBits32(&unencoded_val, read_pos, max_cw_len, - compressed_data); + n_bits = get_n_bits32(&unencoded_val, read_pos, max_cw_len, + compressed_data, info->cmp_size); if (!n_bits) return -1U; if (unencoded_val < info->spill_used && unencoded_val != 0) /* consistency check */ @@ -1135,8 +1150,8 @@ static unsigned int decode_multi(const void *compressed_data, /*TODO: debug message */ return -1U; } - n_bits = GetNBits32(&unencoded_val, read_pos, unencoded_len, - compressed_data); + n_bits = get_n_bits32(&unencoded_val, read_pos, unencoded_len, + compressed_data, info->cmp_size); if (!n_bits) return -1U; @@ -1194,7 +1209,8 @@ static int decode_16(const void *compressed_data, const struct cmp_info *info, decoded_data[i] = (uint16_t)decoded_val; } - if (read_pos != info->cmp_size) { + if (read_pos != info->cmp_size && + cmp_bit_to_4byte(read_pos) != cmp_bit_to_4byte(info->cmp_size)) { debug_print("Warning: The size of the decompressed bitstream does not match the size of the compressed bitstream. Check if the parameters used for decompression are the same as those used for compression.\n"); return 1; } @@ -1207,7 +1223,7 @@ static int decode_S_FX(const void *compressed_data, const struct cmp_info *info, { size_t i; unsigned int read_pos = 0; - struct cmp_info info_exp_flag = *info; + struct cmp_info info_exp_flag; info_exp_flag.golomb_par_used = GOLOMB_PAR_EXPOSURE_FLAGS; @@ -1220,6 +1236,8 @@ static int decode_S_FX(const void *compressed_data, const struct cmp_info *info, if (!decoded_data) return -1; + info_exp_flag = *info; + for (i = 0; i < info->samples_used; i++) { uint32_t decoded_val; diff --git a/lib/rmap.c b/lib/rmap.c index 908b6225ac1f509ae88401323a6195a873d41d8a..465494a30029b3406289b7653970da006f3936f8 100644 --- a/lib/rmap.c +++ b/lib/rmap.c @@ -581,7 +581,7 @@ struct rmap_pkt *rmap_pkt_from_buffer(uint8_t *buf, uint32_t len) if (pkt->data_len) { if (len < RMAP_DATA_START + n + pkt->data_len + 1) { /* +1 for data CRC */ - printf("buffer len is smaller than the contained RMAP packet; buf len: %u bytes vs RMAP: %lu bytes needed\n", + printf("buffer len is smaller than the contained RMAP packet; buf len: %u bytes vs RMAP: %zu bytes needed\n", len , RMAP_DATA_START + n + pkt->data_len); goto error; } diff --git a/test/cmp_tool/cmp_tool_integration_test.py b/test/cmp_tool/cmp_tool_integration_test.py index 60e13cbb96b1a21734340f3157a0ee070976c4e0..925304a0cce9226be87eaf847818234c0746e6c2 100644 --- a/test/cmp_tool/cmp_tool_integration_test.py +++ b/test/cmp_tool/cmp_tool_integration_test.py @@ -2,9 +2,13 @@ import subprocess import shlex import sys import os +import math + +from datetime import datetime +from datetime import timedelta PATH_CMP_TOOL = "./cmp_tool" -VERSION = "0.06" +VERSION = "0.07" EXIT_FAILURE = 1 EXIT_SUCCESS = 0 @@ -41,16 +45,114 @@ def del_file(filePath): print("The file %s could not be deleted because it does not exist!" % (filePath)) +def cuc_timestamp(now): + epoch = datetime(2020, 1, 1) + timestamp = (now - epoch).total_seconds() + + cuc_coarse = int( math.floor(timestamp) * 256 * 256) + cuc_fine = int(math.floor((timestamp - math.floor(timestamp))*256*256)) + + cuc = int(cuc_coarse)+int(cuc_fine) + + return cuc + + +def read_in_cmp_header(compressed_string): + GENERIC_HEADER_SIZE = 30 + + header = { 'asw_version_id' : { 'value': -1, 'bits': 16 }, + 'cmp_ent_size' : { 'value': -1, 'bits': 24 }, + 'original_size' : { 'value': -1, 'bits': 24 }, + 'start_time' : { 'value': -1, 'bits': 48 }, + 'end_timestamp' : { 'value': -1, 'bits': 48 }, + 'data_type' : { 'value': -1, 'bits': 16 }, + 'cmp_mode_used' : { 'value': -1, 'bits': 8 }, + 'model_value_used' : { 'value': -1, 'bits': 8 }, + 'model_id' : { 'value': -1, 'bits': 16 }, + 'model_counter' : { 'value': -1, 'bits': 8 }, + 'spare' : { 'value': 0, 'bits': 8 }, + 'lossy_cmp_par_used' : { 'value': -1, 'bits': 16 }, + 'spill_used' : { 'value': -1, 'bits': 16 }, + 'golomb_par_used' : { 'value': -1, 'bits': 8 }, + 'ap1_spill_used' : { 'value': -1, 'bits': 16 }, + 'ap1_golomb_par' : { 'value': -1, 'bits': 8 }, + 'ap2_spill_used' : { 'value': -1, 'bits': 16 }, + 'ap2_golomb_par' : { 'value': -1, 'bits': 8 }, + 'spill_1_used' : { 'value': -1, 'bits': 24 }, + 'cmp_par_1_used' : { 'value': -1, 'bits': 16 }, + 'spill_2_used' : { 'value': -1, 'bits': 24 }, + 'cmp_par_2_used' : { 'value': -1, 'bits': 16 }, + 'spill_3_used' : { 'value': -1, 'bits': 24 }, + 'cmp_par_3_used' : { 'value': -1, 'bits': 16 }, + 'spill_4_used' : { 'value': -1, 'bits': 24 }, + 'cmp_par_4_used' : { 'value': -1, 'bits': 16 }, + 'spill_5_used' : { 'value': -1, 'bits': 24 }, + 'cmp_par_5_used' : { 'value': -1, 'bits': 16 }, + 'spill_6_used' : { 'value': -1, 'bits': 24 }, + 'cmp_par_6_used' : { 'value': -1, 'bits': 16 }, + 'compressed_data' : { 'value': -1, 'bits': -1 }} + + l = 0 + compressed_string = compressed_string.replace(" ", "").replace("\n", "") # remove white spaces + + # Generic Header + for i, data_field in enumerate(header): + if header[data_field]['bits'] % 4 != 0: + raise Exception("only work with 4 bit aligned data fields") + byte_len = header[data_field]['bits']//4 + header[data_field]['value'] = int(compressed_string[l:l+byte_len], 16) + l += byte_len + # end of the GENERIC_HEADER + if l >= GENERIC_HEADER_SIZE*2: + break + + data_type = header['data_type']['value'] & 0x7FFF + # Imagette Headers + if data_type == 1 or data_type == 2: + for data_field in list(header)[12:18]: + if header[data_field]['bits'] % 4 != 0: + raise Exception("only work with 4 bit aligned data fields") + byte_len = header[data_field]['bits']//4 + header[data_field]['value'] = int(compressed_string[l:l+byte_len], 16) + l += byte_len + # skip adaptive stuff if non adaptive header + if data_field == 'golomb_par_used' and data_type == 1: + l +=4 + break + l += 2 # spare bits + + # Non-Imagette Headers + elif data_type < 24: + for data_field in list(header)[18:30]: + if header[data_field]['bits'] % 4 != 0: + raise Exception("only work with 4 bit aligned data fields") + byte_len = header[data_field]['bits']//4 + header[data_field]['value'] = int(compressed_string[l:l+byte_len], 16) + l += byte_len + else: + raise Exception("data_type unknown") + + header['compressed_data']['value'] = compressed_string[l::] + + # version conversion fuu + version_id = header['asw_version_id']['value'] + if version_id & 0x8000: + header['asw_version_id']['value'] = "%.2f" % (int(version_id & 0x7F00)/256. + int(version_id&0xFF)*0.01) + + return header + + HELP_STRING = \ """usage: %s [options] [<argument>] General Options: -h, --help Print this help text and exit - -V, --version Print program version and exit - -v, --verbose Print various debugging information + -o <prefix> Use the <prefix> for output files -n, --model_cfg Print a default model configuration and exit --diff_cfg Print a default 1d-differencing configuration and exit + --no_header Do not add a compression entity header in front of the compressed data -a, --rdcu_par Add additional RDCU control parameters - -o <prefix> Use the <prefix> for output files + -V, --version Print program version and exit + -v, --verbose Print various debugging information Compression Options: -c <file> File containing the compressing configuration -d <file> File containing the data to be compressed @@ -58,9 +160,9 @@ Compression Options: --rdcu_pkt Generate RMAP packets for an RDCU compression --last_info <.info file> Generate RMAP packets for an RDCU compression with parallel read of the last results Decompression Options: - -i <file> File containing the decompression information -d <file> File containing the compressed data -m <file> File containing the model of the compressed data + -i <file> File containing the decompression information (required if --no_header was used) Guessing Options: --guess <mode> Search for a good configuration for compression <mode> -d <file> File containing the data to be compressed @@ -73,8 +175,14 @@ CMP_START_STR = \ """######################################################### ### PLATO Compression/Decompression Tool Version %s ### ######################################################### +Info: Note that the behaviour of the cmp_tool has changed. From now on, the compressed data will be preceded by a header by default. The old behaviour can be achieved with the --no_header option. """ % (VERSION) +print(CMP_START_STR) + +CMP_START_STR_CMP = CMP_START_STR + "## Starting the compression ##\n" +CMP_START_STR_DECMP = CMP_START_STR + "## Starting the decompression ##\n" +CMP_START_STR_GUESS = CMP_START_STR + "## Search for a good set of compression parameters ##\n" def test_no_option(): @@ -93,14 +201,14 @@ def test_invalid_option(): if arg == '-q': if sys.platform == 'linux': assert(stderr == "%s: invalid option -- 'q'\n" % (PATH_CMP_TOOL)) - elif "win" in sys.platform: + elif sys.platform == 'win32' or sys.platform == 'cygwin': assert(stderr == "%s: unknown option -- q\n" % (PATH_CMP_TOOL)) else: assert(stderr == "cmp_tool: invalid option -- q\n") else: if sys.platform == 'linux': assert(stderr == "%s: unrecognized option '--not_used'\n" % (PATH_CMP_TOOL)) - elif "win" in sys.platform: + elif sys.platform == 'win32' or sys.platform == 'cygwin': assert(stderr == "%s: unknown option -- not_used\n" % (PATH_CMP_TOOL)) else: assert(stderr == "cmp_tool: unrecognized option `--not_used'\n") @@ -181,7 +289,7 @@ def test_print_model_cfg(): def test_no_data_file_name(): - returncode, stdout, stderr = call_cmp_tool("-d no_data.data") + returncode, stdout, stderr = call_cmp_tool("-d no_data.data --no_header") assert(returncode == EXIT_FAILURE) assert(stdout == CMP_START_STR) assert(stderr == "cmp_tool: No configuration file (-c option) or decompression information file (-i option) specified.\n") @@ -197,11 +305,18 @@ def test_no_c_or_i_file(): def test_no_cfg_exist_test(): returncode, stdout, stderr = call_cmp_tool("-d no_data.dat -c no_cfg.cfg") assert(returncode == EXIT_FAILURE) - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_CMP + "Importing configuration file no_cfg.cfg ... FAILED\n") assert(stderr == "cmp_tool: no_cfg.cfg: No such file or directory\n") +def test_too_many_arg(): + returncode, stdout, stderr = call_cmp_tool("argument") + assert(returncode == EXIT_FAILURE) + assert(stdout == "cmp_tool: To many arguments.\n"+HELP_STRING) + assert(stderr == "") + + def test_compression_diff(): # generate test data data = '00 01 00 02 00 03 00 04 00 05 \n' @@ -219,51 +334,83 @@ def test_compression_diff(): assert(stderr == "") f.write(stdout) - # compression - returncode, stdout, stderr = call_cmp_tool( - " -c "+cfg_file_name+" -d "+data_file_name + " -o "+output_prefix) + add_args = [" --no_header", ""] + for add_arg in add_args: + # compression + returncode, stdout, stderr = call_cmp_tool( + " -c "+cfg_file_name+" -d "+data_file_name + " -o "+output_prefix+add_arg) - # check compression results - assert(returncode == EXIT_SUCCESS) - assert(stderr == "") - assert(stdout == CMP_START_STR + - "Importing configuration file %s ... DONE\n" % (cfg_file_name) + - "Importing data file %s ... \n" % (data_file_name) + - "No samples parameter set. Use samples = 5.\n" + - "... DONE\n" + - "No buffer_length parameter set. Use buffer_length = 12 as compression buffer size.\n" + - "Compress data ... DONE\n" + - "Write compressed data to file %s.cmp ... DONE\n" % (output_prefix) + - "Write decompression information to file %s.info ... DONE\n" % (output_prefix)) - # check compressed data - with open(output_prefix+".cmp", encoding='utf-8') as f: - assert(f.read() == "44 44 40 00 \n") - # check info file - with open(output_prefix+".info", encoding='utf-8') as f: - info = parse_key_value(f.read()) - assert(info['cmp_mode_used'] == '2') - # assert(info['model_value_used'] == '8') # not needed for diff - assert(info['round_used'] == '0') - assert(info['spill_used'] == '60') - assert(info['golomb_par_used'] == '7') - assert(info['samples_used'] == '5') - assert(info['cmp_size'] == '20') - assert(info['cmp_err'] == '0') + # check compression results + assert(returncode == EXIT_SUCCESS) + assert(stderr == "") + assert(stdout == CMP_START_STR_CMP + + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + + "Importing data file %s ... \n" % (data_file_name) + + "No samples parameter set. Use samples = 5.\n" + + "... DONE\n" + + "No buffer_length parameter set. Use buffer_length = 12 as compression buffer size.\n" + + "Compress data ... DONE\n" + + "Write compressed data to file %s.cmp ... DONE\n" % (output_prefix) + + "%s" % ((lambda arg : "Write decompression information to file %s.info ... DONE\n" % (output_prefix) + if arg == " --no_header" else "")(add_arg)) + ) + # check compressed data + cmp_data = "44 44 40 00 \n" + with open(output_prefix+".cmp", encoding='utf-8') as f: + if add_arg == " --no_header": + # check compressed data file + assert(f.read() == "44 44 40 00 \n") + # check info file + with open(output_prefix+".info", encoding='utf-8') as f: + info = parse_key_value(f.read()) + assert(info['cmp_mode_used'] == '2') + # assert(info['model_value_used'] == '8') # not needed for diff + assert(info['round_used'] == '0') + assert(info['spill_used'] == '60') + assert(info['golomb_par_used'] == '7') + assert(info['samples_used'] == '5') + assert(info['cmp_size'] == '20') + assert(info['cmp_err'] == '0') + else: + header = read_in_cmp_header(f.read()) + assert(header['asw_version_id']['value'] == VERSION) + assert(header['cmp_ent_size']['value'] == 36+4) + assert(header['original_size']['value'] == 10) + # todo + assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow())) + #todo + assert(header['end_timestamp']['value'] < cuc_timestamp(datetime.utcnow())) + assert(header['data_type']['value'] == 1) + assert(header['cmp_mode_used']['value'] == 2) + # assert(header['model_value_used']['value'] == 8) + assert(header['model_id']['value'] == 53264) + assert(header['model_counter']['value'] == 0) + assert(header['lossy_cmp_par_used']['value'] == 0) + assert(header['spill_used']['value'] == 60) + assert(header['golomb_par_used']['value'] == 7) + assert(header['compressed_data']['value'] == "44444000") + + # decompression + if add_arg == " --no_header": + returncode, stdout, stderr = call_cmp_tool( + " -i "+output_prefix+".info -d "+output_prefix+".cmp -o "+output_prefix) + else: + returncode, stdout, stderr = call_cmp_tool("-d "+output_prefix+".cmp -o "+output_prefix) - # decompression - returncode, stdout, stderr = call_cmp_tool( - " -i "+output_prefix+".info -d "+output_prefix+".cmp -o "+output_prefix) + # check decompression + assert(stderr == "") + assert(stdout == CMP_START_STR_DECMP + + "%s" % ((lambda arg : "Importing decompression information file %s.info ... DONE\n" % (output_prefix) + if "--no_header" in arg else "")(add_arg)) + + "Importing compressed data file %s.cmp ... DONE\n" % (output_prefix) + + "%s" % ((lambda arg : "Parse the compression entity header ... DONE\n" + if not "--no_header" in arg else "")(add_arg)) + + "Decompress data ... DONE\n" + + "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix)) - # check decompression - assert(returncode == EXIT_SUCCESS) - assert(stdout == CMP_START_STR + - "Importing decompression information file %s.info ... DONE\n" % (output_prefix) + - "Importing compressed data file %s.cmp ... DONE\n" % (output_prefix) + - "Decompress data ... DONE\n" + - "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix)) - assert(stderr == "") - with open(output_prefix+".dat", encoding='utf-8') as f: - assert(f.read() == data) # decompressed data == input data + assert(returncode == EXIT_SUCCESS) + with open(output_prefix+".dat", encoding='utf-8') as f: + assert(f.read() == data) # decompressed data == input data # clean up finally: @@ -274,7 +421,7 @@ def test_compression_diff(): del_file(output_prefix+'.dat') -def test_model_compression(): +def test_model_compression_no_header(): # generate test data data = '00 01 00 02 00 03 00 04 00 05 \n' model = '00 00 00 01 00 02 00 03 00 04 \n' @@ -303,12 +450,12 @@ def test_model_compression(): # compression returncode, stdout, stderr = call_cmp_tool( - " -c "+cfg_file_name+" -d "+data_file_name + " -m "+model_file_name+" -o "+output_prefix1) + " -c "+cfg_file_name+" -d "+data_file_name + " -m "+model_file_name+" -o "+output_prefix1+" --no_header") # check compression results assert(returncode == EXIT_SUCCESS) assert(stderr == "") - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_CMP + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + "Importing data file %s ... DONE\n" % (data_file_name) + "Importing model file %s ... DONE\n" % (model_file_name) + @@ -335,7 +482,7 @@ def test_model_compression(): returncode, stdout, stderr = call_cmp_tool( " -i "+output_prefix1+".info -d "+output_prefix1+".cmp -m "+model_file_name+" -o "+output_prefix2) assert(returncode == EXIT_SUCCESS) - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_DECMP + "Importing decompression information file %s.info ... DONE\n" % (output_prefix1) + "Importing compressed data file %s.cmp ... DONE\n" % (output_prefix1) + "Importing model file %s ... DONE\n" % (model_file_name) + @@ -364,6 +511,99 @@ def test_model_compression(): del_file(output_prefix2+'_upmodel.dat') +def test_raw_mode_compression(): + data = '00 01 00 02 00 03 00 04 00 05 \n' + cfg = "cmp_mode = 0\n" + "samples = 5\n" + "buffer_length = 6\n" + + cfg_file_name = 'raw.cfg' + data_file_name = 'raw.data' + output_prefix = 'raw_compression' + args = ["-c %s -d %s -o %s --no_header" %(cfg_file_name, data_file_name, output_prefix), + "-c %s -d %s -o %s " %(cfg_file_name, data_file_name, output_prefix)] + + try: + with open(cfg_file_name, 'w') as f: + f.write(cfg) + with open(data_file_name, 'w') as f: + f.write(data) + + for arg in args: + returncode, stdout, stderr = call_cmp_tool(arg) + assert(stderr == "") + assert(returncode == EXIT_SUCCESS) + assert(stdout == CMP_START_STR_CMP + + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + + "Importing data file %s ... DONE\n" % (data_file_name) + + "Compress data ... DONE\n" + + "Write compressed data to file %s.cmp ... DONE\n" % (output_prefix) + + "%s" % ((lambda arg : "Write decompression information to file %s.info ... DONE\n" % (output_prefix) + if "--no_header" in arg else "")(arg)) + ) + + # check compressed data + with open(output_prefix+".cmp", encoding='utf-8') as f: + if "--no_header" in arg: + # check compressed data file + assert(f.read() == data[:-1]+"00 00 \n") + # check info file + with open(output_prefix+".info", encoding='utf-8') as f: + info = parse_key_value(f.read()) + assert(info['cmp_mode_used'] == '0') + # assert(info['model_value_used'] == '8') # not needed for diff + assert(info['round_used'] == '0') + # assert(info['spill_used'] == '60') + # assert(info['golomb_par_used'] == '7') + assert(info['samples_used'] == '5') + assert(info['cmp_size'] == '80') + assert(info['cmp_err'] == '0') + else: + header = read_in_cmp_header(f.read()) + assert(header['asw_version_id']['value'] == VERSION) + assert(header['cmp_ent_size']['value'] == 36+12) + assert(header['original_size']['value'] == 10) + # todo + assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow())) + #todo + assert(header['end_timestamp']['value'] < cuc_timestamp(datetime.utcnow())) + assert(header['data_type']['value'] == 1+0x8000) + # assert(header['cmp_mode_used']['value'] == 2) + # assert(header['model_value_used']['value'] == 8) + assert(header['model_id']['value'] == 53264) + assert(header['model_counter']['value'] == 0) + assert(header['lossy_cmp_par_used']['value'] == 0) + # assert(header['spill_used']['value'] == 60) + # assert(header['golomb_par_used']['value'] == 7) + assert(header['compressed_data']['value'] == data[:-1].replace(" ","")+"0000") + + # decompression + if "--no_header" in arg: + returncode, stdout, stderr = call_cmp_tool( + " -i "+output_prefix+".info -d "+output_prefix+".cmp -o "+output_prefix) + else: + returncode, stdout, stderr = call_cmp_tool("-d "+output_prefix+".cmp -o "+output_prefix) + + + # check decompression + assert(stderr == "") + assert(stdout == CMP_START_STR_DECMP + + "%s" % ((lambda arg : "Importing decompression information file %s.info ... DONE\n" % (output_prefix) + if "--no_header" in arg else "")(arg)) + + "Importing compressed data file %s.cmp ... DONE\n" % (output_prefix) + + "%s" % ((lambda arg : "Parse the compression entity header ... DONE\n" + if not "--no_header" in arg else "")(arg)) + + "Decompress data ... DONE\n" + + "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix)) + assert(returncode == EXIT_SUCCESS) + with open(output_prefix+".dat", encoding='utf-8') as f: + assert(f.read() == data) # decompressed data == input data + finally: + del_file(cfg_file_name) + del_file(data_file_name) + del_file("%s.cmp" %(output_prefix)) + del_file("%s.info" %(output_prefix)) + del_file("%s.dat" %(output_prefix)) + + def test_guess_option(): # generate test data data = '00 01 00 01 00 01 00 01 00 01 \n' @@ -371,15 +611,14 @@ def test_guess_option(): data_file_name = 'data.dat' model_file_name = 'model.dat' args = [ - ['guess_RDCU_diff', '--guess RDCU -d %s -o guess --rdcu_par' % + ['guess_RDCU_diff', '--guess RDCU -d %s -o guess --rdcu_par --no_header' % (data_file_name)], ['guess_RDCU_model', '--guess RDCU -d %s -m %s -o guess --rdcu_par' % (data_file_name, model_file_name)], ['guess_level_3', '--guess MODE_DIFF_MULTI --guess_level 3 -d %s -o guess' % (data_file_name)], - ['cfg_folder_not_exist', '--guess RDCU -d ' + - data_file_name+' -o not_exist/guess'], - ['guess_level_10', '--guess MODE_DIFF_MULTI --guess_level 10 -d %s -o guess' % + ['cfg_folder_not_exist', '--guess RDCU -d ' + data_file_name+' -o not_exist/guess'], + ['guess_level_not_supported', '--guess MODE_DIFF_MULTI --guess_level 10 -d %s -o guess' % (data_file_name)], ['guess_unknown_mode', '--guess MODE_UNKNOWN -d %s -o guess' % (data_file_name)], @@ -402,14 +641,16 @@ def test_guess_option(): exp_out = ('', '2', '', '7.27') elif sub_test == 'guess_RDCU_model': exp_out = ( - 'Importing model file model.dat ... DONE\n', '2', '', '5.33') + 'Importing model file model.dat ... DONE\n', '2', '', '0.23') + #cmp_size:15bit-> 4byte cmp_data + 40byte header -> 16bit*5/(44Byte*8) '5.33' elif sub_test == 'guess_level_3': exp_out = ( - '', '3', ' 0%... 6%... 13%... 19%... 25%... 32%... 38%... 44%... 50%... 57%... 64%... 72%... 80%... 88%... 94%... 100%', '11.43') + '', '3', ' 0%... 6%... 13%... 19%... 25%... 32%... 38%... 44%... 50%... 57%... 64%... 72%... 80%... 88%... 94%... 100%', '0.25') #11.43 + # cmp_size:7 bit -> 4byte cmp_data + 36 byte header -> 16bit*5/(40Byte*8) else: exp_out = ('', '', '') - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_GUESS + "Importing data file %s ... \n" % (data_file_name) + "No samples parameter set. Use samples = 5.\n" "... DONE\n" @@ -458,16 +699,16 @@ def test_guess_option(): assert( stderr == "cmp_tool: not_exist/guess.cfg: No such file or directory\n") assert(returncode == EXIT_FAILURE) - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_GUESS + "Importing data file %s ... \n" % (data_file_name) + "No samples parameter set. Use samples = 5.\n" + "... DONE\n" "Search for a good set of compression parameters (level: 2) ... DONE\n" + "Write the guessed compression configuration to file not_exist/guess.cfg ... FAILED\n") - elif sub_test == 'guess_level_10': + elif sub_test == 'guess_level_not_supported': assert(stderr == "cmp_tool: guess level not supported!\n") assert(returncode == EXIT_FAILURE) - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_GUESS + "Importing data file %s ... \n" % (data_file_name) + "No samples parameter set. Use samples = 5.\n" + "... DONE\n" @@ -476,7 +717,7 @@ def test_guess_option(): assert( stderr == "cmp_tool: Error: unknown compression mode: MODE_UNKNOWN\n") assert(returncode == EXIT_FAILURE) - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_GUESS + "Importing data file %s ... \n" % (data_file_name) + "No samples parameter set. Use samples = 5.\n" + "... DONE\n" @@ -485,7 +726,7 @@ def test_guess_option(): assert( stderr == "cmp_tool: Error: model mode needs model data (-m option)\n") assert(returncode == EXIT_FAILURE) - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_GUESS + "Importing data file %s ... \n" % (data_file_name) + "No samples parameter set. Use samples = 5.\n" + "... DONE\n" @@ -519,17 +760,298 @@ def test_small_buf_err(): f.write(key + ' = ' + str(value) + '\n') returncode, stdout, stderr = call_cmp_tool(arg) - assert(stdout == CMP_START_STR + + assert(stdout == CMP_START_STR_CMP + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + "Importing data file %s ... DONE\n" % (data_file_name) + "Compress data ... \n" "Compression error 0x01\n" "... FAILED\n") - assert(stderr == "cmp_tool: the buffer for the compressed data is too small. Try a larger buffer_length parameter.\n") + # assert(stderr == "cmp_tool: the buffer for the compressed data is too small. Try a larger buffer_length parameter.\n") + assert(stderr == "Error: The buffer for the compressed data is too small to hold the compressed data. Try a larger buffer_length parameter.\n") + assert(returncode == EXIT_FAILURE) + + finally: + del_file(data_file_name) + del_file(cfg_file_name) + + +def test_empty_file(): + data = '' + data_file_name = 'empty.dat' + cfg_file_name = 'my.cfg' + arg = '-c %s -d %s' % (cfg_file_name, data_file_name) + + try: + with open(data_file_name, 'w') as f: + f.write(data) + + # create a compression file + with open(cfg_file_name, 'w') as f: + returncode, stdout, stderr = call_cmp_tool("--diff_cfg") + assert(returncode == EXIT_SUCCESS) + assert(stderr == "") + f.write(stdout) + + returncode, stdout, stderr = call_cmp_tool(arg) + assert(stdout == CMP_START_STR_CMP + + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + + "Importing data file %s ... FAILED\n" % (data_file_name)) + assert(stderr == "cmp_tool: %s: Warning: The file is empty.\n" % (data_file_name)) + assert(returncode == EXIT_FAILURE) + + finally: + del_file(data_file_name) + del_file(cfg_file_name) + + +def test_empty_cfg_and_info(): + data = '00 01 0F 02 00 0C 00 42 00 23 \n' + empty_file_name = 'empty' + args = ['-c %s -d %s' % (empty_file_name, empty_file_name), + '-i %s -d %s' % (empty_file_name, empty_file_name)] + + try: + with open(empty_file_name, 'w') as f: + f.write(data) + + for i, arg in enumerate(args): + returncode, stdout, stderr = call_cmp_tool(arg) + if i == 0: + assert(stdout == CMP_START_STR_CMP + + "Importing configuration file %s ... FAILED\n" % (empty_file_name)) + assert(stderr == "cmp_tool: Some parameters are missing. Check " + "if the following parameters: cmp_mode, golomb_par, " + "spill, samples and buffer_length are all set in the " + "configuration file.\n") + else: + assert(stdout == CMP_START_STR_DECMP + + "Importing decompression information file %s ... FAILED\n" % (empty_file_name)) + assert(stderr == "cmp_tool: Some parameters are missing. Check " + "if the following parameters: cmp_mode_used, " + "golomb_par_used, spill_used and samples_used " + "are all set in the information file.\n") + assert(returncode == EXIT_FAILURE) + + finally: + del_file(empty_file_name) + + +def test_wrong_formart_data_fiel(): + data = '42 Wrong format!' + cfg = ("golomb_par = 1\n" + "spill = 4\n" + "cmp_mode = 2\n" + + "samples = 5\n" + "buffer_length = 5\n") + data_file_name = 'wrong.data' + cfg_file_name = 'wrong.cfg' + + try: + with open(data_file_name, 'w') as f: + f.write(data) + with open(cfg_file_name, 'w') as f: + f.write(cfg) + + returncode, stdout, stderr = call_cmp_tool('-c %s -d %s' % (cfg_file_name, data_file_name)) + assert(stdout == CMP_START_STR_CMP + + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + + "Importing data file %s ... FAILED\n" % (data_file_name)) + assert(stderr == "cmp_tool: wrong.data: Error: The data are not correct formatted. Expected format is like: 12 AB 23 CD .. ..\n") assert(returncode == EXIT_FAILURE) + finally: + del_file(data_file_name) + del_file(cfg_file_name) + + +def test_wrong_formart_cmp_fiel(): + cmp_data = 'Wrong format!' + info = ("cmp_size = 32\n" + "golomb_par_used = 1\n" + "spill_used = 4\n" + + "cmp_mode_used = 1\n" +"samples_used=5\n") + cmp_file_name = 'wrong.cmp' + info_file_name = 'wrong.info' + + try: + with open(cmp_file_name, 'w') as f: + f.write(cmp_data) + with open(info_file_name, 'w') as f: + f.write(info) + + returncode, stdout, stderr = call_cmp_tool('-i %s -d %s' % (info_file_name, cmp_file_name)) + assert(stdout == CMP_START_STR_DECMP + + "Importing decompression information file %s ... DONE\n" % (info_file_name) + + "Importing compressed data file %s ... FAILED\n" % (cmp_file_name)) + assert(stderr == "cmp_tool: wrong.cmp: Error: The data are not correct formatted. Expected format is like: 12 AB 23 CD .. ..\n") + assert(returncode == EXIT_FAILURE) + finally: + del_file(cmp_file_name) + del_file(info_file_name) + + +def test_sample_used_is_to_big(): + cmp_data_header = '80 07 00 00 28 00 00 0E 03 9E 1D 5A 67 76 03 9E 1D 5A 67 9F 00 01 02 08 42 23 13 00 00 00 00 3C 07 00 00 00 44 44 44 00 \n' + # ^ wrong samples_used + cmp_data_no_header = '44 44 44 00 \n' + info = ("cmp_size = 20\n" + "golomb_par_used = 1\n" + "spill_used = 4\n" + + "cmp_mode_used = 2\n" +"samples_used=5\n") + cmp_file_name = 'big_cmp_size.cmp' + info_file_name = 'big_cmp_size.info' + pars = ['-i %s -d %s --no_header' % (info_file_name, cmp_file_name), + '-d %s' % (cmp_file_name)] + + try: + with open(info_file_name, 'w') as f: + f.write(info) + + for par in pars: + with open(cmp_file_name, 'w') as f: + if '--no_header' in par: + f.write(cmp_data_no_header) + else: + f.write(cmp_data_header) + + returncode, stdout, stderr = call_cmp_tool(par) + + if '--no_header' in par: + assert(stdout == CMP_START_STR_DECMP + + "Importing decompression information file %s ... DONE\n" % (info_file_name) + + "Importing compressed data file %s ... DONE\n" % (cmp_file_name) + + "Decompress data ... FAILED\n") + else: + assert(stdout == CMP_START_STR_DECMP + + "Importing compressed data file %s ... DONE\n" % (cmp_file_name) + + "Parse the compression entity header ... DONE\n" + + "Decompress data ... FAILED\n") + + assert(stderr == "Error: Buffer overflow detected.\n" + + "Error: Compressed values could not be decoded.\n") + + assert(returncode == EXIT_FAILURE) + finally: + del_file(cmp_file_name) + del_file(info_file_name) + + +def test_read_in_header(): + cmp_file_name = 'read_in_header.cmp' + + # packet wrong formatted + header = '80 07 00 00 Wr on g! \n' + try: + with open(cmp_file_name, 'w') as f: + f.write(header) + returncode, stdout, stderr = call_cmp_tool('-d %s' % (cmp_file_name)) + assert(returncode == EXIT_FAILURE) + assert(stdout == CMP_START_STR_DECMP + + "Importing compressed data file %s ... FAILED\n" % (cmp_file_name)) + assert(stderr == "cmp_tool: %s: Error: The data are not correct formatted. Expected format is like: 12 AB 23 CD .. ..\n" % (cmp_file_name)) + + # packet to small + header = '80 07 00 00 28 00 00 0C 03 9E 1D 5A 67 76 03 9E 1D 5A 67 9F 00 01 02 08 42 23 13 00 00 00 00 3C 07 00 00 00 44 \n' + # packet cut-off ^^ ^^ + with open(cmp_file_name, 'w') as f: + f.write(header) + returncode, stdout, stderr = call_cmp_tool('-d %s' % (cmp_file_name)) + assert(returncode == EXIT_FAILURE) + assert(stdout == CMP_START_STR_DECMP + + "Importing compressed data file %s ... FAILED\n" % (cmp_file_name)) + assert(stderr == "cmp_tool: %s: The size of the compression entity set in the header of the compression entity is not the same size as the read-in file has.\n" %(cmp_file_name)) + + # packet false data type + header = '80 07 00 00 28 00 00 0C 03 9E 1D 5A 67 76 03 9E 1D 5A 67 9F FF FF 02 08 42 23 13 00 00 00 00 3C 07 00 00 00 44 44 44 00 \n' + # ^^ ^^ false data type + with open(cmp_file_name, 'w') as f: + f.write(header) + returncode, stdout, stderr = call_cmp_tool('-d %s' % (cmp_file_name)) + assert(returncode == EXIT_FAILURE) + assert(stdout == CMP_START_STR_DECMP + + "Importing compressed data file %s ... FAILED\n" % (cmp_file_name)) + assert(stderr == "cmp_tool: %s: Error: Compression data type is not supported.\n" % (cmp_file_name)) + + # packet false data type + header = '80 07 00 00 28 00 00 0C 03 9E 1D 5A 67 76 03 9E 1D 5A 67 9F 00 01 FF 08 42 23 13 00 00 00 00 3C 07 00 00 00 44 44 44 00 \n' + # ^^ false cmp_mode_used + with open(cmp_file_name, 'w') as f: + f.write(header) + returncode, stdout, stderr = call_cmp_tool('-d %s' % (cmp_file_name)) + assert(returncode == EXIT_FAILURE) + assert(stdout == CMP_START_STR_DECMP + + "Importing compressed data file %s ... DONE\n" % (cmp_file_name) + + "Parse the compression entity header ... FAILED\n" ) + assert(stderr == "Error: Compression mode not supported.\n") + + finally: + del_file(cmp_file_name) + + + +def test_model_fiel_erros(): + # generate test data + data = '00 01 00 02 00 03 00 04 00 05 \n' + model = '00 00 00 01 00 02 00 03 \n' + cfg = ("golomb_par = 1\n" + "spill = 4\n" + "cmp_mode = 1\n" + + "samples = 5\n" + "buffer_length = 5\n") + data_file_name = 'data_model_err.dat' + cfg_file_name = 'model_err.cfg' + output_prefix = 'model_err_test' + model_file_name = 'model.dat' + + try: + with open(data_file_name, 'w', encoding='utf-8') as f: + f.write(data) + with open(cfg_file_name, 'w', encoding='utf-8') as f: + f.write(cfg) + + # no -m option in model mode + returncode, stdout, stderr = call_cmp_tool( + " -c "+cfg_file_name+" -d "+data_file_name + " -o "+output_prefix+" --no_header") + assert(returncode == EXIT_FAILURE) + assert(stdout == CMP_START_STR_CMP + + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + + "Importing data file %s ... DONE\n" % (data_file_name) + + "Importing model file ... FAILED\n" ) + assert(stderr == "cmp_tool: No model file (-m option) specified.\n") + + # model file to small + with open(model_file_name, 'w', encoding='utf-8') as f: + f.write(model) + returncode, stdout, stderr = call_cmp_tool( + " -c "+cfg_file_name+" -d "+data_file_name + " -m "+model_file_name+" -o "+output_prefix) + assert(returncode == EXIT_FAILURE) + assert(stdout == CMP_START_STR_CMP + + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + + "Importing data file %s ... DONE\n" % (data_file_name) + + "Importing model file %s ... FAILED\n" % (model_file_name) ) + assert(stderr == "cmp_tool: %s: Error: The files do not contain enough data as requested.\n" % (model_file_name)) + + # updated model can not write + with open(model_file_name, 'w', encoding='utf-8') as f: + f.write(data) + + output_prefix = ("longlonglonglonglonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong") + returncode, stdout, stderr = call_cmp_tool( + " -c "+cfg_file_name+" -d "+data_file_name + " -m "+model_file_name+" -o "+output_prefix) + assert(returncode == EXIT_FAILURE) + print(stdout) + assert(stdout == CMP_START_STR_CMP + + "Importing configuration file %s ... DONE\n" % (cfg_file_name) + + "Importing data file %s ... DONE\n" % (data_file_name) + + "Importing model file %s ... DONE\n" % (model_file_name) + + "Compress data ... DONE\n" + + "Write compressed data to file %s.cmp ... DONE\n" %(output_prefix) + + "Write updated model to file %s_upmodel.dat ... FAILED\n" %(output_prefix)) + assert(stderr == "cmp_tool: %s_upmodel.dat: File name too long\n" % (output_prefix)) + # finally: del_file(data_file_name) del_file(cfg_file_name) + del_file(model_file_name) + del_file(output_prefix+".cmp") + del_file(output_prefix+"_upmodel.dat") + # TODO: random test +# TODO: random test +# TODO: random test +