diff --git a/CHANGELOG.md b/CHANGELOG.md index d31d308a30f23084a15e47b4e6c73da6427f282f..f74a06c7db46a58d1de9780de5e0f7722cc58da0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,13 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [0.06] - 12-11-2021 - if samples = 0 the cmp_tool counts the samples in the data file and uses them - if buffer_length = 0 the 2*samples parameter is used as buffer_length +- added feature to guess the compression configuration +- some code restructuring +- added more detailed error messages +- small bug fixes ## [0.05] - 23-04-2021 ### Removed - discard old file format. Now the only input format is like: 12 AB 23 CD .. .. diff --git a/Makefile b/Makefile index cadef099255fb93e4d72b770953d999cd3a13aa5..28c93b0de9d9d273e1bd1656a0996b0b389bbd14 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,11 @@ CC = gcc SOURCEDIR = lib INCLUDEDIR = include BUILDDIR = ./ -CFLAGS := -Wall -Wextra -std=gnu99 -pedantic -Wshadow +CFLAGS := -Wall -Wextra -std=gnu99 -pedantic -Wshadow \ + -Wunreachable-code #-Wdocumentation #-Wsign-conversion RELCFLAGS := -O2 # Release flags DBCFLAGS := -O0 -g3 #debug flags +COVFLAGS := -fprofile-arcs -ftest-coverage #coverage flags CPPFLAGS := -I $(INCLUDEDIR) LDFLAGS := -lm SOURCES := cmp_tool.c \ @@ -18,6 +20,7 @@ SOURCES := cmp_tool.c \ $(SOURCEDIR)/cmp_icu.c \ $(SOURCEDIR)/decmp.c \ $(SOURCEDIR)/rdcu_pkt_to_file.c \ + $(SOURCEDIR)/cmp_guess.c \ $(SOURCEDIR)/cmp_tool_lib.c TARGET := cmp_tool @@ -34,3 +37,13 @@ all: $(SOURCES) debug: $(SOURCES) $(CC) $(CPPFLAGS) $(CFLAGS) $(DBCFLAGS) $^ -o $(TARGET) $(LDFLAGS) + +integration: $(SOURCES) + $(CC) $(CPPFLAGS) $(CFLAGS) $(DBCFLAGS) $(COVFLAGS) $^ -o $(TARGET) $(LDFLAGS) + pytest test/cmp_tool/cmp_tool_integration_test.py + lcov --rc lcov_branch_coverage=1 --capture --directory ./ --output-file coverage.info + genhtml --rc lcov_branch_coverage=1 --branch-coverage coverage.info --output-directory out +#firefox out/index.html + +clean: + rm -rf $(TARGET) *.gcno *.gcda coverage.info out diff --git a/README.md b/README.md index 440ec46bc5b22ef158c1ed69886ab0f6f0829724..66378e4532ab428e264fa94f5d54744bb6e66d51 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,139 @@ # PLATO Compression/Decompression Tool -If you find a bug or have a feature request please file an [issue][1] or send me an [email][2] -Compiled executable can be found [here](https://gitlab.phaidra.org/loidoltd15/cmp_tool/-/tags). +If you find a bug or have a feature request please file an [issue][1] or send +me an [email][2]. +Compiled executables can be found [here][3]. ## Usage -**usage:** `./cmp_tool [options] [<argument>]` -### General Options -| Options | Description | -| ----------------- | ---------------------------------------------------------------------------------- | -| `-h, --help` | Print this help text and exit | -| `-V, --version` | Print program version and exit | -| `-v, --verbose` | Print various debugging information | -| `-n, --model_cfg` | Print a default model configuration and exit<sup>[1](#myfootnote2)</sup> | -| `--diff_cfg` | Print a default 1d-differencing configuration and exit<sup>[1](#myfootnote2)</sup> | -| `-a, --rdcu_par` | Print additional RDCU control parameters | -| `-o <prefix>` | Use the `<prefix>` for output files<sup>[2](#myfootnote2)</sup> | - -<a name="myfootnote1">1</a>) **NOTE:** In the default configurations the samples and buffer_length parameter is set to 0! -<a name="myfootnote2">2</a>) **NOTE:** If the -o option is not used the `<prefix>` will be set to "OUTPUT". - -### Compression Options -| Options | Description | -| --------------------------- | ----------------------------------------------------------------------------------- | -| `-c <file>` | File containing the compressing configuration | -| `-d <file>` | File containing the data to be compressed | -| `-m <file>` | File containing the model of the data to be compressed | -| `--rdcu_pkt` | Generate RMAP packets for a RDCU compression<sup>[3](#myfootnote3)</sup> | -| `--last_info <.info file>` | Generate RMAP packets for a RDCU compression with parallel read of the last results | - -<a name="myfootnote2">3</a>) **NOTE:** When using the `--rdcu_pkt` option the configuration of the RMAP parameters - can be found in the `.rdcu_pkt_mode_cfg file`. The generated packets can be - found in the `TC_FILES` directory. - -### Decompression Options + +**usage:** `cmp_tool [options] [<argument>]` + +### General Options + +| Options | Description | +|:------------------|:------------------------------------------------------------------------------| +| `-h, --help` | Print some help text and exit | +| `-V, --version` | Print program version and exit | +| `-v, --verbose` | Print various debugging information | +| `-n, --model_cfg` | Print a default model configuration and exit<sup>[1](#fnote1)</sup> | +| `--diff_cfg` | Print a default 1d-differencing configuration and exit<sup>[1](#fnote1)</sup> | +| `-a, --rdcu_par` | Add additional RDCU control parameters | +| `-o <prefix>` | Use the `<prefix>` for output files<sup>[2](#fnote2)</sup> | + +<a name="fnote1">1</a>) **NOTE:** In the default configurations the **samples** +and **buffer_length** parameter is set to **0**! +<a name="fnote2">2</a>) **NOTE:** If the -o option is not used the `<prefix>` +will be set to "OUTPUT". + +### Compression Options + +| Options | Description | +|:----------------------------|:-------------------------------------------------------------------------------------| +| `-c <file>` | File containing the compressing configuration | +| `-d <file>` | File containing the data to be compressed | +| `-m <file>` | File containing the model of the data to be compressed | +| `--rdcu_pkt` | Generate RMAP packets for an RDCU compression<sup>[3](#fnoot3)</sup> | +| `--last_info <.info file>` | Generate RMAP packets for an RDCU compression with parallel read of the last results | + +<a name="foot3">3</a>) **NOTE:** When using the `--rdcu_pkt` option the +configuration of the RMAP parameters can be found in the `.rdcu_pkt_mode_cfg file`. +The generated packets can be found in the `TC_FILES` directory. + +### Decompression Options + | Options | Description | -| ----------- | ------------------------------------------------ | +|:----------- |:-------------------------------------------------| | `-i <file>` | File containing the decompression information | -| `-d <file>` | File containing the compressed data | -| `-m <file>` | File containing the model of the compressed data | +| `-d <file>` | File containing the compressed data | +| `-m <file>` | File containing the model of the compressed data | + +### Guessing Options + +| Options | Description | +|:------------------------|:--------------------------------------------------------------------------------| +| `--guess <mode>` | Search for a good configuration for compression \<mode\><sup>[4](#fnoot4)</sup> | +| `-d <file>` | File containing the data to be compressed | +| `-m <file>` | File containing the model of the data to be compressed | +| `--guess_level <level>` | Set guess level to \<level\> (optional)<sup>[5](#fnoot5)</sup> | + +<a name="fnoot4">4</a>) **NOTE:** \<mode\> can be either the compression mode +number or the keyword: `RDCU`. The RDCU mode automatically selects the correct +RDCU-compatible compression mode depending on if the Model (-m) option is set. +<a name="fnoot5">5</a>) **Supported levels:** -### Data Formart -The input data must be formatted as two hex numbers separated by a space. +| guess level | Description | +|:------------|:--------------------------------| +| `1` | fast mode (not implemented yet) | +| `2` | default mode | +| `3` | brute force | + +**Example of Compression Parmeter guessing:** + +``./cmp_tool --guess RDCU -d test_data/test_data1.dat -o myguess`` + +This command creates the file `myguess.cfg` with the guessed compression parameters. + +### Data Format + +The input data must be formatted as two hex numbers separated by a space. For example: `12 AB 34 CD`. ## How to use the tool + A simple example to show how the compression tool works. -0. Run `make` to build the tool +0. Download the [tool][3] or run `make` to build the tool -1. Create a configuration file -* Create a cfg directory +1. Create a configuration file +* Create a cfg directory `mkdir cfg` -* To create a default 1d-differencing mode configuration: +* To create a default 1d-differencing mode configuration: `./cmp_tool --diff_cfg > cfg/default_config_1d.cfg` -* To create a default model mode configuration: +* To create a default model mode configuration: `./cmp_tool --model_cfg > cfg/default_config_model.cfg` -* Change the the `samples` and `buffer_length` parameters from `0` to `50` in the `default_config_1d.cfg` and `default_config_model.cfg` files +* Change the the **`samples`** and **`buffer_length`** parameters from `0` to `50` +in the `default_config_1d.cfg` and `default_config_model.cfg` files -2. Compress data with the default configurations: -* Create a directory for the compressed data. - `mkdir compressed` -* 1d-differencing mode compression +2. Compress data with the default configurations: +* Create a directory for the compressed data + `mkdir compressed` +* 1d-differencing mode compression `./cmp_tool -c cfg/default_config_1d.cfg -d test_data/test_data1.dat -o compressed/data1` - - This creates this two files in compressed directory: - `data1.cmp ` -> compressed `test_data1.dat` data - `data1.info` -> decompression information for `data1.cmp` - -* Model mode compression + This creates these two files in the compressed directory: + `data1.cmp` -> compressed `test_data1.dat` file + `data1.info` -> decompression information for `data1.cmp` +* Model mode compression `./cmp_tool -c cfg/default_config_model.cfg -d test_data/test_data2.dat -m test_data/test_data1.dat -o compressed/data2` - We use `test_data1.dat`as the first model for `test_data2.dat` + We use `test_data1.dat` as the first model for `test_data2.dat`. - Creates this three files in compressed directory: - `data2.cmp ` -> compressed `test_data3.dat` data + Creates these three files in the compressed directory: + `data2.cmp ` -> compressed `test_data3.dat` file `data2.info` -> decompression information for `data2.cmp` - `data2_upmodel.dat` -> updated model used to **compress** the next data in model mode - -3. Decompress the data -* Create a directory for the decompressed data. - `mkdir decompressed` + `data2_upmodel.dat` -> updated model used to **compress** the next data in model mode -* Decompress `data1.cmp` +3. Decompress the data +* Create a directory for the decompressed data + `mkdir decompressed` +* Decompress `data1.cmp` `./cmp_tool -i compressed/data1.info -d compressed/data1.cmp -o decompressed/test_data1` - -* Decompress `data2.cmp` + Creates this file in the decompressed directory: + `test_data1.dat ` -> decompressed `data1.cmp` file +* Decompress `data2.cmp` `./cmp_tool -i compressed/data2.info -d compressed/data2.cmp -m decompressed/test_data1.dat -o decompressed/test_data2` As for the compression we use `test_data1.dat` as our model for decompression. + Creates these two files in the decompressed directory: + `test_data2.dat ` -> decompressed `data2.cmp` file + `data2_upmodel.dat` -> updated model used to **decompress** the next data in model mode + 4. Bonus: Check if the decompressed data are equal to the original data The moment of truth! `diff test_data/test_data1.dat decompressed/test_data1.dat` - `diff test_data/test_data2.dat decompressed/test_data2.dat` + `diff test_data/test_data2.dat decompressed/test_data2.dat` And also check if the updated model is the same - `diff compressed/data2_upmodel.dat decompressed/test_data2_upmodel.dat` - + `diff compressed/data2_upmodel.dat decompressed/test_data2_upmodel.dat` [1]: <https://gitlab.phaidra.org/loidoltd15/cmp_tool/-/issues> "issues" [2]: <mailto:dominik.loidolt@univie.ac.at?subject=%5BIssue%5D%20Cmd_Tool> "email" +[3]: <https://gitlab.phaidra.org/loidoltd15/cmp_tool/-/releases> "release" \ No newline at end of file diff --git a/cmp_tool.c b/cmp_tool.c index 9bb5d8a7da6a9c96ea942c01f6311c88a2d38b1a..a314b8f5b3b844718a4adbd4a67c1fa0a8d7cfbe 100755 --- a/cmp_tool.c +++ b/cmp_tool.c @@ -19,30 +19,37 @@ * @see Data Compression User Manual PLATO-UVIE-PL-UM-0001 */ +#include <stdio.h> +#include <stdlib.h> #include <limits.h> #include <getopt.h> #include "include/cmp_tool_lib.h" #include "include/cmp_icu.h" -#include "include/cmp_rdcu.h" #include "include/decmp.h" +#include "include/cmp_guess.h" +#include "include/rdcu_pkt_to_file.h" + #define VERSION "0.06" +#define BUFFER_LENGTH_DEF_FAKTOR 2 + /* * For long options that have no equivalent short option, use a * non-character as a pseudo short option, starting with CHAR_MAX + 1. */ enum { - RDCU_PKT_OPTION = CHAR_MAX + 1, - DIFF_CFG_OPTION, - RDCU_PAR_OPTION, + DIFF_CFG_OPTION = CHAR_MAX + 1, + GUESS_OPTION, + GUESS_LEVEL, + RDCU_PKT_OPTION, LAST_INFO, }; -static struct option const long_options[] = { +static const struct option long_options[] = { {"rdcu_par", no_argument, NULL, 'a'}, {"model_cfg", no_argument, NULL, 'n'}, {"help", no_argument, NULL, 'h'}, @@ -50,10 +57,12 @@ static struct option const long_options[] = { {"version", no_argument, NULL, 'V'}, {"rdcu_pkt", no_argument, NULL, RDCU_PKT_OPTION}, {"diff_cfg", no_argument, NULL, DIFF_CFG_OPTION}, - {"last_info", required_argument, NULL, LAST_INFO} + {"guess", required_argument, NULL, GUESS_OPTION}, + {"guess_level", required_argument, NULL, GUESS_LEVEL}, + {"last_info", required_argument, NULL, LAST_INFO}, + {NULL, 0, NULL, 0} }; - /* prefix of the generated output file names */ static const char *output_prefix = DEFAULT_OUTPUT_PREFIX; @@ -72,167 +81,16 @@ static const char *last_info_file_name; static int verbose_en; -/* generate packets to setup a RDCU compression */ -static int gen_rdcu_write_pkts(struct cmp_cfg *cfg) -{ - int error; - - error = init_rmap_pkt_to_file(); - if (error) { - fprintf(stderr, "%s: Read RMAP packet config file .rdcu_pkt_mode_cfg failed.\n", - PROGRAM_NAME); - return -1; - } - - if (last_info_file_name) { - /* generation of packets for parallel read/write RDCU setup */ - struct cmp_info last_info = {0}; - - error = read_cmp_info(last_info_file_name, &last_info, verbose_en); - if (error) { - fprintf(stderr, "%s: %s: Importing last decompression information file failed.\n", - PROGRAM_NAME, last_info_file_name); - return -1; - } - - error = gen_rdcu_parallel_pkts(cfg, &last_info); - if (error) - return -1; - } - - /* generation of packets for non-parallel read/write RDCU setup */ - error = gen_write_rdcu_pkts(cfg); - if (error) - return -1; - - return 0; -} - +/* find a good set of compression parameters for a given dataset */ +static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode, + int guess_level); /* compress the data and write the results to files */ -static int compression(struct cmp_cfg *cfg, struct cmp_info *info) -{ - int error; - uint32_t cmp_size_byte; - - cfg->icu_output_buf = NULL; - - if (rdcu_pkt_mode) { - printf("Generate compression setup packets ... \n"); - error = gen_rdcu_write_pkts(cfg); - if (error) - goto error; - else - printf("... DONE\n"); - } - - printf("Compress data ... "); - - if (cfg->buffer_length == 0) { - cfg->buffer_length = (cfg->samples+1) * BUFFER_LENGTH_DEF_FAKTOR; /* +1 to prevent malloc(0)*/ - printf("No buffer_length parameter set. Use buffer_length = %u as compression buffer size.\n", - cfg->buffer_length); - } - - cfg->icu_output_buf = malloc(cfg->buffer_length * size_of_a_sample(cfg->cmp_mode)); - if (cfg->icu_output_buf == NULL) { - fprintf(stderr, "%s: Error allocating memory for output buffer.\n", PROGRAM_NAME); - goto error; - } - - error = icu_compress_data(cfg, info); - if (error || info->cmp_err != 0) { - printf("Compression error %#X\n", info->cmp_err); - goto error; - } else - printf("DONE\n"); - - if (rdcu_pkt_mode) { - printf("Generate the read results packets ... "); - error = gen_read_rdcu_pkts(info); - if (error) - goto error; - else - printf("DONE\n"); - } - - printf("Write compressed data to file %s.cmp ... ", output_prefix); - /* length of cmp_size in bytes words (round up to 4 bytes) */ - cmp_size_byte = (info->cmp_size + 31)/32 * 4; - error = write_cmp_data_file(cfg->icu_output_buf, cmp_size_byte, - output_prefix, ".cmp", verbose_en); - if (error) - goto error; - else - printf("DONE\n"); - free(cfg->icu_output_buf); - cfg->icu_output_buf = NULL; - - printf("Write decompression information to file %s.info ... ", - output_prefix); - error = write_info(info, output_prefix, print_rdcu_cfg); - if (error) - goto error; - else - printf("DONE\n"); - - if (verbose_en) { - printf("\n"); - print_cmp_info(info); - printf("\n"); - } - - return 0; - -error: - printf("FAILED\n"); - free(cfg->icu_output_buf); - return -1; -} +static int compression(struct cmp_cfg *cfg, struct cmp_info *info); /* decompress the data and write the results in file(s)*/ static int decompression(uint32_t *decomp_input_buf, uint16_t *input_model_buf, - struct cmp_info *info) -{ - int error; - uint16_t *decomp_output; - - printf("Decompress data ... "); - - if (info->samples_used == 0) - return 0; /* nothing to decompress */ - - decomp_output = malloc(info->samples_used * - size_of_a_sample(info->cmp_mode_used)); - if (decomp_output == NULL) { - printf("FAILED\n"); - fprintf(stderr, "%s: Error allocating memory for decompressed data.\n", PROGRAM_NAME); - return -1; - } - - error = decompress_data(decomp_input_buf, input_model_buf, info, - decomp_output); - if (error) { - printf("FAILED\n"); - free(decomp_output); - return -1; - } - else - printf("DONE\n"); - - printf("Write decompressed data to file %s.dat ... ", output_prefix); - - error = write_to_file16(decomp_output, info->samples_used, - output_prefix, ".dat", verbose_en); - free(decomp_output); - if (error) { - printf("FAILED\n"); - return -1; - } else - printf("DONE\n"); - - return 0; -} + struct cmp_info *info); /** @@ -254,10 +112,14 @@ int main(int argc, char **argv) const char *info_file_name = NULL; const char *data_file_name = NULL; const char *model_file_name = NULL; + const char *guess_cmp_mode = NULL; + const char *program_name = argv[0]; + int cmp_operation = 0; int print_model_cfg = 0; + int guess_operation = 0; + int guess_level = DEFAULT_GUESS_LEVEL; int print_diff_cfg = 0; - int cmp_operation = 0; struct cmp_cfg cfg = {0}; /* compressor configuration struct */ struct cmp_info info = {0}; /* decompression information struct */ @@ -268,7 +130,7 @@ int main(int argc, char **argv) /* show help if no arguments are provided */ if (argc < 2) { - Print_Help(*argv); + print_help(program_name); exit(EXIT_FAILURE); } @@ -286,7 +148,7 @@ int main(int argc, char **argv) data_file_name = optarg; break; case 'h': /* --help */ - Print_Help(*argv); + print_help(argv[0]); exit(EXIT_SUCCESS); break; case 'i': @@ -305,12 +167,19 @@ int main(int argc, char **argv) verbose_en = 1; break; case 'V': /* --version */ - printf("%s %s\n", PROGRAM_NAME, VERSION); + printf("%s version %s\n", PROGRAM_NAME, VERSION); exit(EXIT_SUCCESS); break; case DIFF_CFG_OPTION: print_diff_cfg = 1; break; + case GUESS_OPTION: + guess_operation = 1; + guess_cmp_mode = optarg; + break; + case GUESS_LEVEL: + guess_level = atoi(optarg); + break; case RDCU_PKT_OPTION: rdcu_pkt_mode = 1; break; @@ -319,11 +188,39 @@ int main(int argc, char **argv) last_info_file_name = optarg; break; default: - Print_Help(*argv); + print_help(program_name); exit(EXIT_FAILURE); break; } } + argc -= optind; + argv += optind; + + if (argc > 0) { + printf("%s: To many arguments.\n", PROGRAM_NAME); + print_help(argv[0]); + exit(EXIT_FAILURE); + } +#if 0 + if (argc > 0) { + if(!data_file_name) + data_file_name = argv[0]; + else { + printf("You can define the data file using either the -d option or the first argument, but not both.\n"); + print_help(program_name); + exit(EXIT_FAILURE); + } + } + if (argc > 1) { + if(!model_file_name) + model_file_name = argv[1]; + else { + printf("You can define the model file using either the -m option or the second argument, but not both.\n"); + print_help(program_name); + exit(EXIT_FAILURE); + } + } +#endif if (print_model_cfg == 1) { print_cfg(&DEFAULT_CFG_MODEL, print_rdcu_cfg); @@ -340,42 +237,49 @@ int main(int argc, char **argv) VERSION); printf("#########################################################\n\n"); - if (!cfg_file_name && !info_file_name) { - fprintf(stderr, "%s: No configuration file (-c option) or decompression information file (-i option) specified.\n", - PROGRAM_NAME); - exit(EXIT_FAILURE); - } - if (!data_file_name) { fprintf(stderr, "%s: No data file (-d option) specified.\n", PROGRAM_NAME); exit(EXIT_FAILURE); } + if (!cfg_file_name && !info_file_name && !guess_operation) { + fprintf(stderr, "%s: No configuration file (-c option) or decompression information file (-i option) specified.\n", + PROGRAM_NAME); + exit(EXIT_FAILURE); + } + /* read input data and .cfg or .info */ - if (cmp_operation) { /* compression mode */ - printf("### Compression ###\n"); + if (cmp_operation || guess_operation) { /* compression mode */ + if (cmp_operation) { + /* printf("### Compression ###\n"); */ + printf("Importing configuration file %s ... ", cfg_file_name); + error = read_cmp_cfg(cfg_file_name, &cfg, verbose_en); + if (error) + goto fail; + else + printf("DONE\n"); - printf("Importing configuration file %s ... ", cfg_file_name); - error = read_cmp_cfg(cfg_file_name, &cfg, verbose_en); - if (error) - goto fail; - else - printf("DONE\n"); + } + + printf("Importing data file %s ... ", data_file_name); /* count the samples in the data file when samples == 0 */ if (cfg.samples == 0) { size = read_file16(data_file_name, NULL, 0, 0); if (size < 0) goto fail; - /* TODO: check size%size_of_a_sample(cfg.cmp_mode) == 0 */ + if (size%size_of_a_sample(cfg.cmp_mode) != 0) { + fprintf(stderr, "\n%s: %s: Error: The data file is not correct formatted. Expected multiple of %zu hex words.\n", + PROGRAM_NAME, data_file_name, size_of_a_sample(cfg.cmp_mode)); + goto fail; + } + cfg.samples = size/size_of_a_sample(cfg.cmp_mode); - printf("No samples parameter set. Use samples = %u.\n", + printf("\nNo samples parameter set. Use samples = %u.\n... ", cfg.samples); } - printf("Importing data file %s ... ", data_file_name); - cfg.input_buf = malloc(cfg.samples * size_of_a_sample(cfg.cmp_mode)); if (!cfg.input_buf) { fprintf(stderr, "%s: Error allocating memory for input data buffer.\n", PROGRAM_NAME); @@ -392,8 +296,8 @@ int main(int argc, char **argv) } else { /* decompression mode*/ uint32_t cmp_size_byte; - printf("### Decompression ###\n\n"); - printf("Importing decompression information file ... "); + /* printf("### Decompression ###\n\n"); */ + printf("Importing decompression information file %s ... ", info_file_name); error = read_cmp_info(info_file_name, &info, verbose_en); if (error) @@ -418,9 +322,10 @@ int main(int argc, char **argv) printf("DONE\n"); } - /* read in model for compressed or decompression */ + /* read in model */ if ((cmp_operation && model_mode_is_used(cfg.cmp_mode)) || - (!cmp_operation && model_mode_is_used(info.cmp_mode_used))) { + (!cmp_operation && model_mode_is_used(info.cmp_mode_used)) || + (guess_operation && model_file_name)) { uint32_t model_length; if (!model_file_name) { @@ -430,7 +335,7 @@ int main(int argc, char **argv) printf("Importing model file %s ... ", model_file_name); - if (cmp_operation) + if (cmp_operation || guess_operation) model_length = cfg.samples; else model_length = info.samples_used; @@ -447,15 +352,18 @@ int main(int argc, char **argv) else printf("DONE\n"); - if (cmp_operation) + if (cmp_operation || guess_operation) cfg.model_buf = input_model_buf; } - if (cmp_operation) { /* perform a compression */ + if (guess_operation) { + error = guess_cmp_pars(&cfg, guess_cmp_mode, guess_level); + if (error) + goto fail; + } else if (cmp_operation) { /* perform a compression */ error = compression(&cfg, &info); if (error) goto fail; - } else { /* perform a decompression */ error = decompression(decomp_input_buf, input_model_buf, &info); if (error) @@ -463,9 +371,10 @@ int main(int argc, char **argv) } /* write our the updated model for compressed or decompression */ - if ((cmp_operation && model_mode_is_used(cfg.cmp_mode)) || - (!cmp_operation && model_mode_is_used(info.cmp_mode_used))) { - printf("Write updated model to file %s_upmodel.dat ... " , + if (!guess_operation && + ((cmp_operation && model_mode_is_used(cfg.cmp_mode)) || + (!cmp_operation && model_mode_is_used(info.cmp_mode_used)))) { + printf("Write updated model to file %s_upmodel.dat ... ", output_prefix); error = write_to_file16(input_model_buf, info.samples_used, output_prefix, "_upmodel.dat", verbose_en); @@ -475,7 +384,6 @@ int main(int argc, char **argv) printf("DONE\n"); } - free(cfg.input_buf); free(decomp_input_buf); free(input_model_buf); @@ -491,3 +399,208 @@ fail: exit(EXIT_FAILURE); } + + +/* generate packets to setup a RDCU compression */ +static int gen_rdcu_write_pkts(struct cmp_cfg *cfg) +{ + int error; + + error = init_rmap_pkt_to_file(); + if (error) { + fprintf(stderr, "%s: Read RMAP packet config file .rdcu_pkt_mode_cfg failed.\n", + PROGRAM_NAME); + return -1; + } + + if (last_info_file_name) { + /* generation of packets for parallel read/write RDCU setup */ + struct cmp_info last_info = {0}; + + error = read_cmp_info(last_info_file_name, &last_info, verbose_en); + if (error) { + fprintf(stderr, "%s: %s: Importing last decompression information file failed.\n", + PROGRAM_NAME, last_info_file_name); + return -1; + } + + error = gen_rdcu_parallel_pkts(cfg, &last_info); + if (error) + return -1; + } + + /* generation of packets for non-parallel read/write RDCU setup */ + error = gen_write_rdcu_pkts(cfg); + if (error) + return -1; + + return 0; +} + + +/* find a good set of compression parameters for a given dataset */ +static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode, + int guess_level) +{ + int error; + uint32_t cmp_size; + float cr; + + printf("Search for a good set of compression parameters (level: %d) ... ", guess_level); + if (!strcmp(guess_cmp_mode, "RDCU")) { + if (cfg->model_buf) + cfg->cmp_mode = CMP_GUESS_DEF_MODE_MODEL; + else + cfg->cmp_mode = CMP_GUESS_DEF_MODE_DIFF; + } else { + error = cmp_mode_parse(guess_cmp_mode, &cfg->cmp_mode); + if (error) { + fprintf(stderr, "%s: Error: unknown compression mode: %s\n", PROGRAM_NAME, guess_cmp_mode); + return -1; + } + } + if (model_mode_is_used(cfg->cmp_mode) && !cfg->model_buf) { + fprintf(stderr, "%s: Error: model mode needs model data (-m option)\n", PROGRAM_NAME); + return -1; + } + + cmp_size = cmp_guess(cfg, guess_level); + if (!cmp_size) + return -1; + printf("DONE\n"); + + printf("Write the guessed compression configuration to file %s.cfg ... ", output_prefix); + error = write_cfg(cfg, output_prefix, print_rdcu_cfg, verbose_en); + if (error) + return -1; + printf("DONE\n"); + + cr = (8.0 * cfg->samples * size_of_a_sample(cfg->cmp_mode))/cmp_size; + printf("Guessed parameters can compress the data with a CR of %.2f.\n", cr); + + return 0; +} + + +/* compress the data and write the results to files */ +static int compression(struct cmp_cfg *cfg, struct cmp_info *info) +{ + int error; + uint32_t cmp_size_byte; + + cfg->icu_output_buf = NULL; + + if (cfg->buffer_length == 0) { + cfg->buffer_length = (cfg->samples+1) * BUFFER_LENGTH_DEF_FAKTOR; /* +1 to prevent malloc(0)*/ + printf("No buffer_length parameter set. Use buffer_length = %u as compression buffer size.\n", + cfg->buffer_length); + } + + if (rdcu_pkt_mode) { + printf("Generate compression setup packets ...\n"); + error = gen_rdcu_write_pkts(cfg); + if (error) + goto error_cleanup; + else + printf("... DONE\n"); + } + + printf("Compress data ... "); + + cfg->icu_output_buf = malloc(cfg->buffer_length * size_of_a_sample(cfg->cmp_mode)); + if (cfg->icu_output_buf == NULL) { + fprintf(stderr, "%s: Error allocating memory for output buffer.\n", PROGRAM_NAME); + goto error_cleanup; + } + + error = icu_compress_data(cfg, info); + if (error || info->cmp_err != 0) { + printf("\nCompression error 0x%02X\n... ", info->cmp_err); + if ((info->cmp_err >> SMALL_BUFFER_ERR_BIT) & 1U) + fprintf(stderr, "%s: the buffer for the compressed data is too small. Try a larger buffer_length parameter.\n", PROGRAM_NAME); + goto error_cleanup; + } else + printf("DONE\n"); + + if (rdcu_pkt_mode) { + printf("Generate the read results packets ... "); + error = gen_read_rdcu_pkts(info); + if (error) + goto error_cleanup; + else + printf("DONE\n"); + } + + printf("Write compressed data to file %s.cmp ... ", output_prefix); + /* length of cmp_size in bytes words (round up to 4 bytes) */ + cmp_size_byte = (info->cmp_size + 31)/32 * 4; + error = write_cmp_data_file(cfg->icu_output_buf, cmp_size_byte, + output_prefix, ".cmp", verbose_en); + if (error) + goto error_cleanup; + else + printf("DONE\n"); + free(cfg->icu_output_buf); + cfg->icu_output_buf = NULL; + + printf("Write decompression information to file %s.info ... ", + output_prefix); + error = write_info(info, output_prefix, print_rdcu_cfg); + if (error) + goto error_cleanup; + else + printf("DONE\n"); + + if (verbose_en) { + printf("\n"); + print_cmp_info(info); + printf("\n"); + } + + return 0; + +error_cleanup: + free(cfg->icu_output_buf); + return -1; +} + + +/* decompress the data and write the results in file(s)*/ +static int decompression(uint32_t *decomp_input_buf, uint16_t *input_model_buf, + struct cmp_info *info) +{ + int error; + uint16_t *decomp_output; + + printf("Decompress data ... "); + + if (info->samples_used == 0) + return 0; /* nothing to decompress */ + + decomp_output = malloc(info->samples_used * + size_of_a_sample(info->cmp_mode_used)); + if (decomp_output == NULL) { + fprintf(stderr, "%s: Error allocating memory for decompressed data.\n", PROGRAM_NAME); + return -1; + } + + error = decompress_data(decomp_input_buf, input_model_buf, info, + decomp_output); + if (error) { + free(decomp_output); + return -1; + } + printf("DONE\n"); + + printf("Write decompressed data to file %s.dat ... ", output_prefix); + + error = write_to_file16(decomp_output, info->samples_used, + output_prefix, ".dat", verbose_en); + free(decomp_output); + if (error) + return -1; + + printf("DONE\n"); + + return 0; +} diff --git a/include/cmp_guess.h b/include/cmp_guess.h new file mode 100644 index 0000000000000000000000000000000000000000..fe2ed0989a6b8ab188c92a33cb0b43a3ba2414e1 --- /dev/null +++ b/include/cmp_guess.h @@ -0,0 +1,38 @@ +/** + * @file cmp_guess.h + * @author Dominik Loidolt (dominik.loidolt@univie.ac.at), + * @date 2020 + * + * @copyright GPLv2 + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * @brief helps the user to find a good compression parameters for a given + * dataset + */ + +#ifndef CMP_GUESS_H +#define CMP_GUESS_H + +#include "cmp_support.h" + +#define DEFAULT_GUESS_LEVEL 2 + +#define CMP_GUESS_DEF_MODE_DIFF MODE_DIFF_ZERO +#define CMP_GUESS_DEF_MODE_MODEL MODE_MODEL_MULTI + +/* how often the model is updated before it is reset default value */ +#define CMP_GUESS_N_MODEL_UPDATE_DEF 8 + +uint32_t cmp_guess(struct cmp_cfg *cfg, int level); +void cmp_guess_set_model_updates(int n_model_updates); + +uint16_t cmp_guess_model_value(int n_model_updates); + +#endif /* CMP_GUESS_H */ diff --git a/include/cmp_icu.h b/include/cmp_icu.h index a2acd62bef5653127ecd061b2829f4c0eb2d58ab..cd7f193e10ba3e6b89a863c1f259d948f7c0ce8f 100644 --- a/include/cmp_icu.h +++ b/include/cmp_icu.h @@ -25,4 +25,9 @@ int icu_compress_data(struct cmp_cfg *cfg, struct cmp_info *info); + +int cmp_pre_process(struct cmp_cfg *cfg); +int cmp_map_to_pos(struct cmp_cfg *cfg); +uint32_t cmp_encode_data(struct cmp_cfg *cfg); + #endif /* _CMP_ICU_H_ */ diff --git a/include/cmp_support.h b/include/cmp_support.h index 7c49dc8f86d3d961f43c27432313b1234be8e86c..aeafd0df6209cc39b14ed8648ad06d69f748fe29 100644 --- a/include/cmp_support.h +++ b/include/cmp_support.h @@ -62,6 +62,8 @@ #define SAM2BYT \ 2 /* sample to byte conversion factor; one samples has 16 bits (2 bytes) */ +#define CMP_GOOD_SPILL_DIFF_MULTI 2 /* good guess for the spill parameter using the MODE_DIFF_MULTI */ + /** * @brief The cmp_cfg structure can contain the complete configuration of the HW as * well as the SW compressor. @@ -163,6 +165,7 @@ int model_mode_is_used(unsigned int cmp_mode); int diff_mode_is_used(unsigned int cmp_mode); int raw_mode_is_used(unsigned int cmp_mode); int rdcu_supported_mode_is_used(unsigned int cmp_mode); +int cmp_mode_available(unsigned int cmp_mode); int zero_escape_mech_is_used(unsigned int cmp_mode); int multi_escape_mech_is_used(unsigned int cmp_mode); diff --git a/include/cmp_tool_lib.h b/include/cmp_tool_lib.h index b3b8e1de65efccef31f59d48fc1c45e77c412fc6..b8eadaadd57c93fdf3f4e661bdf3db575f4bba09 100644 --- a/include/cmp_tool_lib.h +++ b/include/cmp_tool_lib.h @@ -15,12 +15,9 @@ * more details. */ -#include <stdio.h> -#include <stdlib.h> #include <string.h> #include "cmp_support.h" -#include "rdcu_pkt_to_file.h" #define PROGRAM_NAME "cmp_tool" #define MAX_CONFIG_LINE 256 @@ -29,7 +26,7 @@ #define BUFFER_LENGTH_DEF_FAKTOR 2 -void Print_Help(const char *argv); +void print_help(const char *program_name); int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en); int read_cmp_info(const char *file_name, struct cmp_info *info, int verbose_en); @@ -38,7 +35,7 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int verbose_en); ssize_t read_file16(const char *file_name, uint16_t *buf, uint32_t samples, int verbose_en); -ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t buf_len, +ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t samples, int verbose_en); int write_cmp_data_file(const void *buf, uint32_t buf_size, const char @@ -46,6 +43,9 @@ int write_cmp_data_file(const void *buf, uint32_t buf_size, const char int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char *output_prefix, const char *name_extension, int verbose); int write_info(const struct cmp_info *info, const char *output_prefix, - int machine_cfg); - + int rdcu_cfg); +int write_cfg(const struct cmp_cfg *cfg, const char *output_prefix, int rdcu_cfg, + int verbose); void print_cfg(const struct cmp_cfg *cfg, int rdcu_cfg); + +uint32_t cmp_mode_parse(const char *cmp_mode_str, uint32_t *cmp_mode); diff --git a/lib/cmp_data_types.c b/lib/cmp_data_types.c index 1137bf4e2d3a80177d50e67f7b514564dc54cedb..1df363cb274aa698815432ef3e9ed51557c88d5b 100644 --- a/lib/cmp_data_types.c +++ b/lib/cmp_data_types.c @@ -17,9 +17,6 @@ */ -#include <stddef.h> -#include <stdio.h> - #include "../include/cmp_data_types.h" #include "../include/cmp_support.h" #include "../include/cmp_debug.h" @@ -295,7 +292,7 @@ struct S_FX_EFX add_S_FX_EFX(struct S_FX_EFX a, struct S_FX_EFX b) * @note change the data buffer in-place * @note the exposure_flags are not rounded * - * @param data_buf a S_FX_EFX formatted data buffer + * @param data S_FX_EFX formatted data buffer * @param samples the size of the data buffer measured in S_FX_EFX samples * @param round number of bits to round; if zero no rounding takes place * @@ -318,8 +315,7 @@ int lossy_rounding_S_FX_EFX(struct S_FX_EFX *data, unsigned int samples, if (round == 0) return 0; - for (i = 0; i < samples; i++) - { + for (i = 0; i < samples; i++) { data[i].FX = round_fwd(data[i].FX, round); data[i].EFX = round_fwd(data[i].EFX, round); } @@ -431,8 +427,7 @@ int lossy_rounding_S_FX_NCOB(struct S_FX_NCOB *data_buf, unsigned int samples, if (round == 0) return 0; - for (i = 0; i < samples; i++) - { + for (i = 0; i < samples; i++) { data_buf[i].FX = round_fwd(data_buf[i].FX, round); data_buf[i].NCOB_X = round_fwd(data_buf[i].NCOB_X, round); data_buf[i].NCOB_Y = round_fwd(data_buf[i].NCOB_Y, round); @@ -559,8 +554,7 @@ int lossy_rounding_S_FX_EFX_NCOB_ECOB(struct S_FX_EFX_NCOB_ECOB *data_buf, if (round == 0) /* round 0 means loss less compression, no further processing is necessary */ return 0; - for (i = 0; i < samples; i++) - { + for (i = 0; i < samples; i++) { data_buf[i].FX = round_fwd(data_buf[i].FX, round); data_buf[i].NCOB_X = round_fwd(data_buf[i].NCOB_X, round); data_buf[i].NCOB_Y = round_fwd(data_buf[i].NCOB_Y, round); @@ -587,8 +581,7 @@ int de_lossy_rounding_S_FX_EFX_NCOB_ECOB(struct S_FX_EFX_NCOB_ECOB *data_buf, if (round_used == 0) /* round 0 means loss less compression, no further processing is necessary */ return 0; - for (i = 0; i < samples_used; i++) - { + for (i = 0; i < samples_used; i++) { uint32_t mask = (~0U << (32-round_used)); if (data_buf[i].FX & mask) { diff --git a/lib/cmp_guess.c b/lib/cmp_guess.c new file mode 100644 index 0000000000000000000000000000000000000000..e272cffb07fbea07b831a2c0c80aa36cd4da501d --- /dev/null +++ b/lib/cmp_guess.c @@ -0,0 +1,319 @@ +/** + * @file cmp_guess.c + * @author Dominik Loidolt (dominik.loidolt@univie.ac.at), + * @date 2021 + * + * @copyright GPLv2 + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * @brief helps the user to find a good compression parameters for a given + * dataset + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "../include/cmp_icu.h" +#include "../include/cmp_guess.h" + + +/* how often the model is updated before it is rested */ +static int num_model_updates = CMP_GUESS_N_MODEL_UPDATE_DEF; + + +/** + * @brief sets how often the model is updated before model reset for the + * cmp_guess function + * @note the default value is CMP_GUESS_N_MODEL_UPDATE_DEF + * @note this is needed to guess a good model_value + * + * @param n_model_updates number of model updates + */ + +void cmp_guess_set_model_updates(int n_model_updates) +{ + num_model_updates = n_model_updates; +} + + +/** + * @brief guess a good model value + * + * @param n_model_updates number of model updates + * + * @returns guessed model_value + */ + +uint16_t cmp_guess_model_value(int n_model_updates) +{ + if (n_model_updates <= 2) + return 8; + if (n_model_updates <= 5) + return 10; + if (n_model_updates <= 11) + return 11; + if (n_model_updates <= 21) + return 12; + else + return 13; +} + + +/** + * @brief guess a good configuration with pre_cal_method + * + * @param cfg compression configuration structure + * + * @returns the size in bits of the compressed data of the guessed + * configuration; 0 on error + */ + +static uint32_t pre_cal_method(struct cmp_cfg *cfg) +{ + uint32_t g; + uint32_t cmp_size; + uint32_t cmp_size_best = ~0U; + uint32_t golomb_par_best = 0; + uint32_t spill_best = 0; + + for (g = MIN_RDCU_GOLOMB_PAR; g < MAX_RDCU_GOLOMB_PAR; ++g) { + uint32_t s = cmp_get_good_spill(g, cfg->cmp_mode); + + cfg->golomb_par = g; + cfg->spill = s; + cmp_size = cmp_encode_data(cfg); + if (cmp_size == 0) { + return 0; + } else if (cmp_size < cmp_size_best) { + cmp_size_best = cmp_size; + golomb_par_best = g; + spill_best = s; + } + } + cfg->golomb_par = golomb_par_best; + cfg->spill = spill_best; + + return cmp_size_best; +} + + +/** + * @brief guess a good configuration with brute force method + * + * @param cfg compression configuration structure + * + * @returns the size in bits of the compressed data of the guessed + * configuration; 0 on error + */ + +#define CMP_GUESS_MAX_CAL_STEPS 20274 +static uint32_t brute_force(struct cmp_cfg *cfg) +{ + uint32_t g, s; + uint32_t n_cal_steps = 0, last = 0; + const uint32_t max_cal_steps = CMP_GUESS_MAX_CAL_STEPS; + uint32_t cmp_size; + uint32_t cmp_size_best = ~0U; + uint32_t golomb_par_best = 0; + uint32_t spill_best = 0; + uint32_t percent; + + /* short cut for zero escape mechanism */ + if (zero_escape_mech_is_used(cfg->cmp_mode)) + return pre_cal_method(cfg); + + printf("0%%... "); + fflush(stdout); + + for (g = MIN_RDCU_GOLOMB_PAR; g < MAX_RDCU_GOLOMB_PAR; ++g) { + for (s = MIN_RDCU_SPILL; s < get_max_spill(g, cfg->cmp_mode); ++s) { + cfg->golomb_par = g; + cfg->spill = s; + + cmp_size = cmp_encode_data(cfg); + if (cmp_size == 0) { + return 0; + } else if (cmp_size < cmp_size_best) { + cmp_size_best = cmp_size; + golomb_par_best = g; + spill_best = s; + } + } + n_cal_steps += s; + + percent = n_cal_steps*100/max_cal_steps; + if (percent > 5+last && percent < 100) { + last = percent; + printf("%u%%... ", percent); + fflush(stdout); + } + } + printf("100%% "); + cfg->golomb_par = golomb_par_best; + cfg->spill = spill_best; + + return cmp_size_best; +} + + +/** + * @brief guessed rdcu specific adaptive parameters + * + * @param cfg compression configuration structure + * @note internal use only + */ + +static void add_rdcu_pars_internal(struct cmp_cfg *cfg) +{ + if (cfg->golomb_par == MIN_RDCU_GOLOMB_PAR) { + cfg->ap1_golomb_par = cfg->golomb_par + 1; + cfg->ap2_golomb_par = cfg->golomb_par + 2; + + } else if (cfg->golomb_par == MAX_RDCU_GOLOMB_PAR) { + cfg->ap1_golomb_par = cfg->golomb_par - 2; + cfg->ap2_golomb_par = cfg->golomb_par - 1; + } else { + cfg->ap1_golomb_par = cfg->golomb_par - 1; + cfg->ap2_golomb_par = cfg->golomb_par + 1; + } + + cfg->ap1_spill = cmp_get_good_spill(cfg->ap1_golomb_par, cfg->cmp_mode); + cfg->ap2_spill = cmp_get_good_spill(cfg->ap2_golomb_par, cfg->cmp_mode); + + if (model_mode_is_used(cfg->cmp_mode)) { + cfg->rdcu_data_adr = DEFAULT_CFG_MODEL.rdcu_data_adr; + cfg->rdcu_model_adr = DEFAULT_CFG_MODEL.rdcu_model_adr; + cfg->rdcu_new_model_adr = DEFAULT_CFG_MODEL.rdcu_new_model_adr; + cfg->rdcu_buffer_adr = DEFAULT_CFG_MODEL.rdcu_buffer_adr; + } else { + cfg->rdcu_data_adr = DEFAULT_CFG_DIFF.rdcu_data_adr; + cfg->rdcu_model_adr = DEFAULT_CFG_DIFF.rdcu_model_adr; + cfg->rdcu_new_model_adr = DEFAULT_CFG_DIFF.rdcu_new_model_adr; + cfg->rdcu_buffer_adr = DEFAULT_CFG_DIFF.rdcu_buffer_adr; + } +} + + +/** + * @brief guess a good compression configuration + * @details use the referenced in the cfg struct (samples, input_buf, model_buf + * (optional)) and the cmp_mode to find a good set of compression parameters + * @note compression parameters in the cfg struct (golomb_par, spill, model_value, + * ap1_.., ap2_.., buffer_length, ...) are overwritten by this function + * + * @param cfg compression configuration structure + * @param level guess_level 1 -> fast; 2 -> default; 3 -> slow(brute force) + * + * @returns the size in bits of the compressed data of the guessed + * configuration; 0 on error + */ + +uint32_t cmp_guess(struct cmp_cfg *cfg, int level) +{ + size_t size; + struct cmp_cfg work_cfg; + int err; + uint32_t cmp_size = 0; + + if (!cfg) + return 0; + + if (!cfg->input_buf) + return 0; + + if (!rdcu_supported_mode_is_used(cfg->cmp_mode)) { + printf("This compression mode is not implied yet.\n"); + return 0; + } + /* make a working copy of the input data (and model) because the + * following function works inplace */ + work_cfg = *cfg; + work_cfg.input_buf = NULL; + work_cfg.model_buf = NULL; + work_cfg.icu_new_model_buf = NULL; + work_cfg.icu_output_buf = NULL; + work_cfg.buffer_length = 0; + + size = cfg->samples * size_of_a_sample(work_cfg.cmp_mode); + if (size == 0) + goto error; + work_cfg.input_buf = malloc(size); + if (!work_cfg.input_buf) { + printf("malloc() failed!\n"); + goto error; + } + memcpy(work_cfg.input_buf, cfg->input_buf, size); + + if (cfg->model_buf && model_mode_is_used(cfg->cmp_mode)) { + work_cfg.model_buf = malloc(size); + if (!work_cfg.model_buf) { + printf("malloc() failed!\n"); + goto error; + } + memcpy(work_cfg.model_buf, cfg->model_buf, size); + + work_cfg.icu_new_model_buf = malloc(size); + if (!work_cfg.icu_new_model_buf) { + printf("malloc() failed!\n"); + goto error; + } + } + + err = cmp_pre_process(&work_cfg); + if (err) + goto error; + + err = cmp_map_to_pos(&work_cfg); + if (err) + goto error; + + /* find the best parameters */ + switch (level) { + case 3: + cmp_size = brute_force(&work_cfg); + break; + case 1: + printf("guess level 1 not implied yet use guess level 2\n"); + /* fall through */ + case 2: + cmp_size = pre_cal_method(&work_cfg); + break; + default: + fprintf(stderr, "cmp_tool: guess level not supported!\n"); + goto error; + break; + } + if (!cmp_size) + goto error; + + free(work_cfg.input_buf); + free(work_cfg.model_buf); + free(work_cfg.icu_new_model_buf); + + cfg->golomb_par = work_cfg.golomb_par; + cfg->spill = work_cfg.spill; + + cfg->model_value = cmp_guess_model_value(num_model_updates); + + if (rdcu_supported_mode_is_used(cfg->cmp_mode)) + add_rdcu_pars_internal(cfg); + + cfg->buffer_length = ((cmp_size + 32)&~0x1FU)/(size_of_a_sample(work_cfg.cmp_mode)*8); + + return cmp_size; + +error: + free(work_cfg.input_buf); + free(work_cfg.model_buf); + free(work_cfg.icu_new_model_buf); + return 0; +} + diff --git a/lib/cmp_icu.c b/lib/cmp_icu.c index 1a8d1574c08e6551f5fa4baf429eed9261c60a98..21aabe7d9e444ca494a65f4720a1b2c59f4159f2 100644 --- a/lib/cmp_icu.c +++ b/lib/cmp_icu.c @@ -30,7 +30,6 @@ #include "../include/cmp_debug.h" - /** * @brief check if the compressor configuration is valid for a SW compression, * see the user manual for more information (PLATO-UVIE-PL-UM-0001). @@ -62,9 +61,8 @@ int icu_cmp_cfg_valid(const struct cmp_cfg *cfg, struct cmp_info *info) cfg_invalid++; } - if (cfg->samples == 0) { + if (cfg->samples == 0) debug_print("Warning: The samples parameter is 0. No data are compressed. This behavior may not be intended.\n"); - } /* icu_output_buf can be NULL if rdcu compression is used */ if (cfg->icu_output_buf == NULL) { @@ -226,7 +224,7 @@ static int set_info(struct cmp_cfg *cfg, struct cmp_info *info) if (cfg->model_value > UINT8_MAX) return -1; - if(info) { + if (info) { info->cmp_err = 0; info->cmp_mode_used = (uint8_t)cfg->cmp_mode; info->model_value_used = (uint8_t)cfg->model_value; @@ -310,7 +308,7 @@ static int diff_32(uint32_t *data_buf, unsigned int samples, unsigned int round) * @note change the data_buf in-place * @note output is I_0 = I_0, I_i = I_i - I_i-1, where i is the array index * - * @param data_buf pointer to a S_FX data buffer + * @param data pointer to a S_FX data buffer * @param samples amount of data samples in the data buffer * @param round number of bits to round; if zero no rounding takes place * @@ -344,7 +342,7 @@ static int diff_S_FX(struct S_FX *data, unsigned int samples, unsigned int * @note change the data_buf in-place * @note output is I_0 = I_0, I_i = I_i - I_i-1, where i is the array index * - * @param data_buf pointer to a S_FX_EFX data buffer + * @param data pointer to a S_FX_EFX data buffer * @param samples amount of data samples in the data buffer * @param round number of bits to round; if zero no rounding takes place * @@ -378,7 +376,7 @@ static int diff_S_FX_EFX(struct S_FX_EFX *data, unsigned int samples, unsigned * @note change the data_buf in-place * @note output is I_0 = I_0, I_i = I_i - I_i-1, where i is the array index * - * @param data_buf pointer to a S_FX_NCOB data buffer + * @param data pointer to a S_FX_NCOB data buffer * @param samples amount of data samples in the data buffer * @param round number of bits to round; if zero no rounding takes place * @@ -412,7 +410,7 @@ static int diff_S_FX_NCOB(struct S_FX_NCOB *data, unsigned int samples, unsigned * @note change the data_buf in-place * @note output is I_0 = I_0, I_i = I_i - I_i-1, where i is the array index * - * @param data_buf pointer to a S_FX_EFX_NCOB_ECOB data buffer + * @param data pointer to a S_FX_EFX_NCOB_ECOB data buffer * @param samples amount of data samples in the data buffer * @param round number of bits to round; if zero no rounding takes place * @@ -540,7 +538,6 @@ static int model_32(uint32_t *data_buf, uint32_t *model_buf, unsigned int sample * @note update the model_buf in-place if up_model_buf = NULL * * @param data_buf pointer to the S_FX data buffer to process - * @param model_buf pointer to the model buffer of the data to process * @param model_buf pointer to the updated model buffer (if NULL model_buf * will be overwrite with the updated model) * @param samples amount of data samples in the data_buf and model_buf buffer @@ -607,7 +604,7 @@ int model_S_FX(struct S_FX *data_buf, struct S_FX *model_buf, * @returns 0 on success, error otherwise */ -int pre_process(struct cmp_cfg *cfg) +int cmp_pre_process(struct cmp_cfg *cfg) { if (!cfg) return -1; @@ -934,13 +931,12 @@ static int map_to_pos_S_FX_EFX_NCOB_ECOB(struct S_FX_EFX_NCOB_ECOB *data_buf, * * @note change the data_buf in-place * - * @param data_buf pointer to the data to process - * @param buf_len length of the data to process + * @param cfg configuration contains all parameters required for compression * * @returns 0 on success, error otherwise */ -static int map_to_pos(struct cmp_cfg *cfg) +int cmp_map_to_pos(struct cmp_cfg *cfg) { int zero_mode_used; @@ -1114,7 +1110,7 @@ static encoder_ptr select_encoder(unsigned int golomb_par) * @param nBits number of bits to put in the bitstream * @param destAddr this is the pointer to the beginning of the bitstream * @param dest_len length of the bitstream buffer (starting at destAddr) - * @returns number of bits written, 0 if the number was too big, -1 if the + * @returns TODO number of bits written, 0 if the number was too big, -2 if the * destAddr buffer is to small to store the bitstream * @note works in SRAM2 */ @@ -1127,6 +1123,9 @@ static unsigned int put_n_bits32(unsigned int value, unsigned int bitOffset, unsigned int bitsLeft, shiftRight, shiftLeft, localEndPos; unsigned int mask; + if (!destAddr) + return nBits; + /* check if destination buffer is large enough */ /* TODO: adapt that to the other science products */ if ((bitOffset + nBits) > (((dest_len+1)/2)*2 * 16)) { @@ -1336,9 +1335,8 @@ static int encode_outlier_zero(uint32_t value_to_encode, int max_bits, /* put the data unencoded in the bitstream */ err = put_n_bits32(value_to_encode, enc->cmp_size, max_bits, cfg->icu_output_buf, cfg->buffer_length); - if (err <= 0) { + if (err <= 0) return err; - } enc->cmp_size += max_bits; @@ -1449,23 +1447,18 @@ int encode_value(uint32_t value_to_encode, int bit_len, struct cmp_cfg *cfg, } -static int encode_16(uint16_t *data_to_encode, struct cmp_cfg *cfg, - struct encoder_struct *enc) +static int encode_16(struct cmp_cfg *cfg, struct encoder_struct *enc) { size_t i; + uint16_t *data_to_encode; if (!cfg) return -1; - if (!cfg->samples) - return 0; - - if (!data_to_encode) - return -1; - if (!enc) return -1; + data_to_encode = cfg->input_buf; for (i = 0; i < cfg->samples; i++) { int err = encode_value(data_to_encode[i], 16, cfg, enc); @@ -1614,7 +1607,7 @@ static int encode_S_FX_EFX_NCOB_ECOB(struct cmp_cfg *cfg, struct encoder_struct } /* pad the bitstream with zeros */ -int pad_bitstream(struct cmp_cfg *cfg, uint32_t cmp_size, struct cmp_info *info) +int pad_bitstream(struct cmp_cfg *cfg, uint32_t cmp_size) { int n_bits = 0; @@ -1628,21 +1621,15 @@ int pad_bitstream(struct cmp_cfg *cfg, uint32_t cmp_size, struct cmp_info *info) n_bits = put_n_bits32(0, cmp_size, n_pad_bits, cfg->icu_output_buf, cfg->buffer_length); - if (n_bits <= 0) { - /* the icu_output_buf is to small to store the whole bitstream */ - if (info) { - info->cmp_err |= 1UL << SMALL_BUFFER_ERR_BIT; /* set small buffer error */ - info->cmp_size = 0; - } + if (n_bits <= 0) return -2; - } } } return n_bits; } -static int encode_data(struct cmp_cfg *cfg, struct cmp_info *info) +uint32_t cmp_encode_data(struct cmp_cfg *cfg) { struct encoder_struct enc; int err, n_bits; @@ -1659,7 +1646,7 @@ static int encode_data(struct cmp_cfg *cfg, struct cmp_info *info) case MODE_MODEL_MULTI: case MODE_DIFF_ZERO: case MODE_DIFF_MULTI: - err = encode_16((uint16_t *)cfg->input_buf, cfg, &enc); + err = encode_16(cfg, &enc); break; case MODE_RAW_S_FX: err = encode_raw_S_FX(cfg, &enc); @@ -1704,28 +1691,24 @@ static int encode_data(struct cmp_cfg *cfg, struct cmp_info *info) break; } - if (err == -2) { - /* the icu_output_buf is to small to store the whole bitstream */ - info->cmp_err |= 1UL << SMALL_BUFFER_ERR_BIT; /* set small buffer error */ - return err; - } - if (info) - info->cmp_size = enc.cmp_size; - - n_bits = pad_bitstream(cfg, enc.cmp_size, info); + n_bits = pad_bitstream(cfg, enc.cmp_size); if (n_bits < 0) return n_bits; #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) - { - size_t i; - uint32_t *p = (uint32_t *)cfg->icu_output_buf; + { + size_t i; + uint32_t *p = (uint32_t *)cfg->icu_output_buf; + if (p) for (i = 0; i < (enc.cmp_size + n_bits)/32; i++) cpu_to_be32s(&p[i]); - } + } #endif /*__BYTE_ORDER__ */ - return 0; + if (err) + return err; + else + return enc.cmp_size; } @@ -1756,6 +1739,7 @@ static int encode_data(struct cmp_cfg *cfg, struct cmp_info *info) int icu_compress_data(struct cmp_cfg *cfg, struct cmp_info *info) { int err; + int cmp_size = 0; err = set_info(cfg, info); if (err) @@ -1765,17 +1749,22 @@ int icu_compress_data(struct cmp_cfg *cfg, struct cmp_info *info) if (err) return err; - err = pre_process(cfg); + err = cmp_pre_process(cfg); if (err) return err; - err = map_to_pos(cfg); + err = cmp_map_to_pos(cfg); if (err) return err; - err = encode_data(cfg, info); - if (err) - return err; + cmp_size = cmp_encode_data(cfg); + if (cmp_size == -2 && info) + /* the icu_output_buf is to small to store the whole bitstream */ + info->cmp_err |= 1UL << SMALL_BUFFER_ERR_BIT; /* set small buffer error */ + if (cmp_size < 0) + return cmp_size; + if (info) + info->cmp_size = cmp_size; return 0; } diff --git a/lib/cmp_support.c b/lib/cmp_support.c index 0ad0696a478781c4d223bf24d94d46bf9f08ebc7..6e69b118281a6c31888ff481dee5932008a5faec 100644 --- a/lib/cmp_support.c +++ b/lib/cmp_support.c @@ -17,7 +17,6 @@ * @see Data Compression User Manual PLATO-UVIE-PL-UM-0001 */ -#include <stdio.h> #include "../include/cmp_support.h" #include "../include/cmp_data_types.h" @@ -246,6 +245,26 @@ int rdcu_supported_mode_is_used(unsigned int cmp_mode) } +/** + * @brief check if the mode is available + * + * @param cmp_mode compression mode + * + * @returns 1 when mode is available, otherwise 0 + */ + +int cmp_mode_available(unsigned int cmp_mode) +{ + if (diff_mode_is_used(cmp_mode) || + model_mode_is_used(cmp_mode) || + raw_mode_is_used(cmp_mode)) + return 1; + else + return 0; + +} + + /** * @brief check if zero escape symbol mechanism mode is used * @@ -442,7 +461,7 @@ uint32_t cmp_get_good_spill(unsigned int golomb_par, unsigned int cmp_mode) 317, 324, 330, 336, 344, 351, 358, 363, 370, 377, 383, 391, 397, 405, 411, 418, 424, 431, 452 }; - if (zero_escape_mech_is_used(cmp_mode) || cmp_mode == MODE_DIFF_MULTI) + if (zero_escape_mech_is_used(cmp_mode)) return get_max_spill(golomb_par, cmp_mode); if (cmp_mode == MODE_MODEL_MULTI) { @@ -452,6 +471,9 @@ uint32_t cmp_get_good_spill(unsigned int golomb_par, unsigned int cmp_mode) return LUT_RDCU_MULIT[golomb_par]; } + if (cmp_mode == MODE_DIFF_MULTI) + return CMP_GOOD_SPILL_DIFF_MULTI; + return 0; } @@ -553,7 +575,8 @@ unsigned int size_of_bitstream(unsigned int cmp_size) /** * @brief calculate the need bytes for the model * - * @param cmp_size compressed data size, measured in bits + * @param samples amount of model samples + * @param cmp_mode compression mode * * @returns the size in bytes to store the hole bitstream */ @@ -588,18 +611,16 @@ void print_cmp_cfg(const struct cmp_cfg *cfg) printf("input_buf: %p\n", (void *)cfg->input_buf); if (cfg->input_buf != NULL) { printf("input data:"); - for (i = 0; i < cfg->samples; i++) { + for (i = 0; i < cfg->samples; i++) printf(" %04X", ((uint16_t *)cfg->input_buf)[i]); - } printf("\n"); } printf("rdcu_data_adr: 0x%06X\n", cfg->rdcu_data_adr); printf("model_buf: %p\n", (void *)cfg->model_buf); if (cfg->model_buf != NULL) { printf("model data:"); - for (i = 0; i < cfg->samples; i++) { + for (i = 0; i < cfg->samples; i++) printf(" %04X", ((uint16_t *)cfg->model_buf)[i]); - } printf("\n"); } printf("rdcu_model_adr: 0x%06X\n", cfg->rdcu_model_adr); diff --git a/lib/cmp_tool_lib.c b/lib/cmp_tool_lib.c index cdaf63d563669da6137c58e6b1d164ec1f4309c2..8c3226be18e483feb59c652b42375fa6e4e85388 100644 --- a/lib/cmp_tool_lib.c +++ b/lib/cmp_tool_lib.c @@ -31,30 +31,35 @@ /** * @brief print help information * - * @param argv argument vector + * @param program_name name of the program */ -void Print_Help(const char *argv) +void print_help(const char *program_name) { - printf("usage: %s [options] [<argument>]\n", &argv[0]); + printf("usage: %s [options] [<argument>]\n", program_name); printf("General Options:\n"); printf(" -h, --help Print this help text and exit\n"); printf(" -V, --version Print program version and exit\n"); printf(" -v, --verbose Print various debugging information\n"); printf(" -n, --model_cfg Print a default model configuration and exit\n"); printf(" --diff_cfg Print a default 1d-differencing configuration and exit\n"); - printf(" -a, --rdcu_par Print additional RDCU control parameters\n"); + printf(" -a, --rdcu_par Add additional RDCU control parameters\n"); printf(" -o <prefix> Use the <prefix> for output files\n"); printf("Compression Options:\n"); printf(" -c <file> File containing the compressing configuration\n"); printf(" -d <file> File containing the data to be compressed\n"); printf(" -m <file> File containing the model of the data to be compressed\n"); - printf(" --rdcu_pkt Generate RMAP packets for a RDCU compression\n"); + printf(" --rdcu_pkt Generate RMAP packets for an RDCU compression\n"); + printf(" --last_info <.info file> Generate RMAP packets for an RDCU compression with parallel read of the last results\n"); printf("Decompression Options:\n"); printf(" -i <file> File containing the decompression information\n"); printf(" -d <file> File containing the compressed data\n"); printf(" -m <file> File containing the model of the compressed data\n"); - printf("\n"); + printf("Guessing Options:\n"); + printf(" --guess <mode> Search for a good configuration for compression <mode>\n"); + printf(" -d <file> File containing the data to be compressed\n"); + printf(" -m <file> File containing the model of the data to be compressed\n"); + printf(" --guess_level <level> Set guess level to <level> (optional)\n"); } @@ -72,14 +77,13 @@ void Print_Help(const char *argv) static FILE *open_file(const char *dirname, const char *filename) { - FILE *fp; char *pathname; int n; if (!dirname) return NULL; - if(!filename) + if (!filename) return NULL; errno = 0; @@ -104,9 +108,7 @@ static FILE *open_file(const char *dirname, const char *filename) abort(); } - fp = fopen(pathname, "w"); - free(pathname); - return fp; + return fopen(pathname, "w"); } @@ -144,7 +146,7 @@ int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char } for (i = 0; i < buf_len; i++) { - fprintf(fp, "%02X %02X",buf[i] >> 8 ,buf[i] & 0xFF); + fprintf(fp, "%02X %02X", buf[i] >> 8, buf[i] & 0xFF); if ((i + 1) % 16 == 0) fprintf(fp, "\n"); else @@ -157,7 +159,7 @@ int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char if (verbose) { printf("\n\n"); for (i = 0; i < buf_len; i++) { - printf("%02X %02X",buf[i] >> 8 ,buf[i] & 0xFF); + printf("%02X %02X", buf[i] >> 8, buf[i] & 0xFF); if ((i + 1) % 16 == 0) printf("\n"); else @@ -185,12 +187,12 @@ int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char * @returns 0 on success, error otherwise */ -int write_cmp_data_file(const void *buf, uint32_t buf_size, const char - *output_prefix, const char *name_extension, int verbose) +int write_cmp_data_file(const void *buf, uint32_t buf_size, const char *output_prefix, + const char *name_extension, int verbose) { unsigned int i; FILE *fp; - uint8_t *p = (uint8_t *)buf; + const uint8_t *p = (const uint8_t *)buf; if (!buf) abort(); @@ -326,7 +328,7 @@ static int sram_addr_to_int(const char *addr) * @see https://eternallyconfuzzled.com/atoi-is-evil-c-learn-why-atoi-is-awful */ -static int atoui32(const char *dep_str , const char *val_str, uint32_t *red_val) +static int atoui32(const char *dep_str, const char *val_str, uint32_t *red_val) { long temp; char *end = NULL; @@ -340,14 +342,14 @@ static int atoui32(const char *dep_str , const char *val_str, uint32_t *red_val) if (!red_val) return -1; + errno = 0; temp = strtol(val_str, &end, 10); - if (end != val_str && errno != ERANGE && temp >= 0 && temp <= - UINT32_MAX) { + if (end != val_str && errno != ERANGE && temp >= 0 && + temp <= UINT32_MAX) { *red_val = (uint32_t)temp; return 0; } else { - fprintf(stderr, "%s: Error read in %s.\n", PROGRAM_NAME, - dep_str); + fprintf(stderr, "%s: Error read in %s.\n", PROGRAM_NAME, dep_str); *red_val = 0; return -1; } @@ -355,14 +357,64 @@ static int atoui32(const char *dep_str , const char *val_str, uint32_t *red_val) /** -* @brief parse a file containing a compressing configuration -* -* @param fp FILE pointer -* @param cfg compression configuration structure holding the read in -* configuration -* -* @returns 0 on success, error otherwise -*/ + * @brief parse a compression mode vale string to a integer + * @note string can be either a number or the name of the compression mode + * + * @param cmp_mode_str string containing the compression mode value to parse + * @param cmp_mode address where the parsed result is written + * + * @returns 0 on success, error otherwise + */ + +uint32_t cmp_mode_parse(const char *cmp_mode_str, uint32_t *cmp_mode) +{ + size_t j; + static const struct { + uint32_t cmp_mode; + const char *str; + } conversion[] = { + {MODE_RAW, "MODE_RAW"}, + {MODE_MODEL_ZERO, "MODE_MODEL_ZERO"}, + {MODE_DIFF_ZERO, "MODE_DIFF_ZERO"}, + {MODE_MODEL_MULTI, "MODE_MODEL_MULTI"}, + {MODE_DIFF_MULTI, "MODE_DIFF_MULTI"}, + }; + + if (!cmp_mode_str) + return -1; + if (!cmp_mode) + return -1; + + if (isalpha(cmp_mode_str[0])) { /* check if mode is given as text */ + for (j = 0; j < sizeof(conversion) / sizeof(conversion[0]); ++j) { + if (!strcmp(cmp_mode_str, conversion[j].str)) { + *cmp_mode = conversion[j].cmp_mode; + return 0; + } + } + return -1; + } else { + if (atoui32(cmp_mode_str, cmp_mode_str, cmp_mode)) + return -1; + } + + if (!cmp_mode_available(*cmp_mode)) + return -1; + + return 0; +} + + +/** + * @brief parse a file containing a compressing configuration + * @note internal use only! + * + * @param fp FILE pointer + * @param cfg compression configuration structure holding the read in + * configuration + * + * @returns 0 on success, error otherwise + */ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg) { @@ -379,7 +431,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg) if (line[0] == '#' || line[0] == '\n') continue; /* skip #'ed or empty lines */ - if (!strchr(line, '\n')){ /* detect a to long line */ + if (!strchr(line, '\n')) { /* detect a to long line */ fprintf(stderr, "%s: Error read in line to long. Maximal line length is %d characters.\n", PROGRAM_NAME, MAX_CONFIG_LINE-1); return -1; @@ -548,8 +600,8 @@ int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en) abort(); if (strstr(file_name, ".info")) { - fprintf(stderr, "%s: %s: .info file extension found on configuration file. You may have selected the wrong file.\n", - PROGRAM_NAME, file_name); + fprintf(stderr, "%s: %s: .info file extension found on configuration file. You may have selected the wrong file.\n", + PROGRAM_NAME, file_name); } fp = fopen(file_name, "r"); @@ -562,9 +614,8 @@ int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en) err = parse_cfg(fp, cfg); fclose(fp); - if (err) { + if (err) return err; - } if (verbose_en) { printf("\n\n"); @@ -584,6 +635,7 @@ int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en) /** * @brief parse a file containing a decompression information + * @note internal use only! * * @param fp FILE pointer * @param info decompression information structure holding the read in @@ -607,7 +659,7 @@ static int parse_info(FILE *fp, struct cmp_info *info) if (line[0] == '#' || line[0] == '\n') continue; /* skip #'ed or empty lines */ - if (!strchr(line, '\n')){ /* detect a to long line */ + if (!strchr(line, '\n')) { /* detect a to long line */ fprintf(stderr, "%s: Error read in line to long. Maximal line length is %d characters.\n", PROGRAM_NAME, MAX_CONFIG_LINE-1); return -1; @@ -740,8 +792,7 @@ static int parse_info(FILE *fp, struct cmp_info *info) int i = sram_addr_to_int(token2); if (i < 0) { - fprintf(stderr,"%s: Error read in " - "rdcu_cmp_adr_used\n", + fprintf(stderr, "%s: Error read in rdcu_cmp_adr_used\n", PROGRAM_NAME); return -1; } @@ -788,10 +839,9 @@ int read_cmp_info(const char *file_name, struct cmp_info *info, int verbose_en) if (!info) abort(); - if (strstr(file_name, ".cfg")) { - fprintf(stderr, "%s: %s: .cfg file extension found on decompression information file. You may have selected the wrong file.\n", - PROGRAM_NAME, file_name); - } + if (strstr(file_name, ".cfg")) + fprintf(stderr, "%s: %s: .cfg file extension found on decompression information file. You may have selected the wrong file.\n", + PROGRAM_NAME, file_name); fp = fopen(file_name, "r"); if (fp == NULL) { @@ -834,7 +884,7 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int ver { /* This is a rather slow implementation. Performance can be improved by * reading larger chunks */ - size_t n; + ssize_t n; FILE *fp; char tmp_str[4]; /* 4 = 2 hex digit's + 1 white space + 1 \0 */ @@ -850,7 +900,7 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int ver /* if buf is NULL we count the words in the file */ if (!buf) - n_word = ~0; + n_word = ~0U; for (n = 0; n < n_word; ) { int j; @@ -912,7 +962,7 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int ver /* print data read in */; if (verbose_en) { - if (n ==0) + if (n == 0) printf("\n\n"); printf("%02X", buf[n]); if (n && !((n + 1) % 32)) @@ -926,15 +976,12 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int ver } if (buf) { - fgets(tmp_str, sizeof(tmp_str), fp); - /* have all data read from file */ - if (!feof(fp) && tmp_str[0] == '\n') - fgets(tmp_str, sizeof(tmp_str), fp); /* read last line break */ - - if (!feof(fp)) { - fprintf(stderr, "%s: %s: Warning: The file may contain more data than specified by the samples parameter.\n", - PROGRAM_NAME, file_name); - } + /* check if there is some additional stuff at the end of the file */ + int c = getc(fp); + if (c != EOF) + if (c != '\n' && getc(fp) != EOF) + fprintf(stderr, "%s: %s: Warning: The file may contain more data than specified by the samples or cmp_size parameter.\n", + PROGRAM_NAME, file_name); } fclose(fp); @@ -968,7 +1015,8 @@ error: ssize_t read_file16(const char *file_name, uint16_t *buf, uint32_t samples, int verbose_en) { - ssize_t size = read_file8(file_name, (uint8_t *)buf, samples*2, verbose_en); + ssize_t size = read_file8(file_name, (uint8_t *)buf, + samples*sizeof(uint16_t), verbose_en); if (size < 0) return size; @@ -981,7 +1029,7 @@ ssize_t read_file16(const char *file_name, uint16_t *buf, uint32_t samples, if (buf) { size_t i; - for (i=0; i < samples; i++) + for (i = 0; i < samples; i++) be16_to_cpus(&buf[i]); } @@ -1005,10 +1053,11 @@ ssize_t read_file16(const char *file_name, uint16_t *buf, uint32_t samples, * @returns the size in bytes to store the file content; negative on error */ -ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t buf_len, +ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t samples, int verbose_en) { - ssize_t size = read_file8(file_name, (uint8_t *)buf, buf_len*4, verbose_en); + ssize_t size = read_file8(file_name, (uint8_t *)buf, + samples*sizeof(uint32_t), verbose_en); if (size < 0) return -1; @@ -1021,7 +1070,7 @@ ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t buf_len, if (buf) { size_t i; - for (i=0; i < buf_len; i++) + for (i = 0; i < samples; i++) be32_to_cpus(&buf[i]); } @@ -1030,123 +1079,169 @@ ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t buf_len, /** - * @brief prints config struct + * @brief write compression configuration to a stream + * @note internal use only! * + * @param fp FILE pointer * @param cfg configuration to print * @param rdcu_cfg if set additional RDCU parameter are printed as well */ -void print_cfg(const struct cmp_cfg *cfg, int rdcu_cfg) +static void write_cfg_internal(FILE *fp, const struct cmp_cfg *cfg, int rdcu_cfg) { - printf("#-------------------------------------------------------------------------------\n"); - printf("# Default Configuration File\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Selected compression mode\n"); - printf("# 0: raw mode\n"); - printf("# 1: model mode with zero escape symbol mechanism\n"); - printf("# 2: 1d differencing mode without input model with zero escape symbol mechanism\n"); - printf("# 3: model mode with multi escape symbol mechanism\n"); - printf("# 4: 1d differencing mode without input model multi escape symbol mechanism\n"); - printf("\n"); - printf("cmp_mode = %u\n", cfg->cmp_mode); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Number of samples (16 bit value) to compress, length of the data and model buffer\n"); - printf("\n"); - printf("samples = %u\n", cfg->samples); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Length of the compressed data buffer in number of samples (16 bit values)\n"); - printf("\n"); - printf("buffer_length = %u\n", cfg->buffer_length); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Golomb parameter for dictionary selection\n"); - printf("\n"); - printf("golomb_par = %u\n", cfg->golomb_par); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Spillover threshold for encoding outliers\n"); - printf("\n"); - printf("spill = %u\n", cfg->spill); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Model weighting parameter\n"); - printf("\n"); - printf("model_value = %u\n", cfg->model_value); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Number of noise bits to be rounded\n"); - printf("\n"); - printf("round = %u\n", cfg->round); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Default Configuration File\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Selected compression mode\n"); + fprintf(fp, "# 0: raw mode\n"); + fprintf(fp, "# 1: model mode with zero escape symbol mechanism\n"); + fprintf(fp, "# 2: 1d differencing mode without input model with zero escape symbol mechanism\n"); + fprintf(fp, "# 3: model mode with multi escape symbol mechanism\n"); + fprintf(fp, "# 4: 1d differencing mode without input model multi escape symbol mechanism\n"); + fprintf(fp, "\n"); + fprintf(fp, "cmp_mode = %u\n", cfg->cmp_mode); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Number of samples (16 bit value) to compress, length of the data and model buffer\n"); + fprintf(fp, "\n"); + fprintf(fp, "samples = %u\n", cfg->samples); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Length of the compressed data buffer in number of samples (16 bit values)\n"); + fprintf(fp, "\n"); + fprintf(fp, "buffer_length = %u\n", cfg->buffer_length); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Golomb parameter for dictionary selection\n"); + fprintf(fp, "\n"); + fprintf(fp, "golomb_par = %u\n", cfg->golomb_par); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Spillover threshold for encoding outliers\n"); + fprintf(fp, "\n"); + fprintf(fp, "spill = %u\n", cfg->spill); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Model weighting parameter\n"); + fprintf(fp, "\n"); + fprintf(fp, "model_value = %u\n", cfg->model_value); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Number of noise bits to be rounded\n"); + fprintf(fp, "\n"); + fprintf(fp, "round = %u\n", cfg->round); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); if (rdcu_cfg) { - printf("#-------------------------------------------------------------------------------\n"); - printf("# Hardware Compressor Settings (not need for SW compression)\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("\n"); - printf("# Adaptive 1 Golomb parameter; HW only\n"); - printf("\n"); - printf("ap1_golomb_par = %u\n", cfg->ap1_golomb_par); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Adaptive 1 spillover threshold; HW only\n"); - printf("\n"); - printf("ap1_spill = %u\n", cfg->ap1_spill); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Adaptive 2 Golomb parameter; HW only\n"); - printf("\n"); - printf("ap2_golomb_par = %u\n", cfg->ap2_golomb_par); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# Adaptive 2 spillover threshold; HW only\n"); - printf("\n"); - printf("ap2_spill = %u\n", cfg->ap2_spill); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# RDCU data to compress start address, the first data address in the RDCU SRAM; HW only\n"); - printf("\n"); - printf("rdcu_data_adr = 0x%06X\n", + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Hardware Compressor Settings (not need for SW compression)\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "\n"); + fprintf(fp, "# Adaptive 1 Golomb parameter; HW only\n"); + fprintf(fp, "\n"); + fprintf(fp, "ap1_golomb_par = %u\n", cfg->ap1_golomb_par); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Adaptive 1 spillover threshold; HW only\n"); + fprintf(fp, "\n"); + fprintf(fp, "ap1_spill = %u\n", cfg->ap1_spill); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Adaptive 2 Golomb parameter; HW only\n"); + fprintf(fp, "\n"); + fprintf(fp, "ap2_golomb_par = %u\n", cfg->ap2_golomb_par); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# Adaptive 2 spillover threshold; HW only\n"); + fprintf(fp, "\n"); + fprintf(fp, "ap2_spill = %u\n", cfg->ap2_spill); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# RDCU data to compress start address, the first data address in the RDCU SRAM; HW only\n"); + fprintf(fp, "\n"); + fprintf(fp, "rdcu_data_adr = 0x%06X\n", cfg->rdcu_data_adr); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# RDCU model start address, the first model address in the RDCU SRAM\n"); - printf("\n"); - printf("rdcu_model_adr = 0x%06X\n", + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# RDCU model start address, the first model address in the RDCU SRAM\n"); + fprintf(fp, "\n"); + fprintf(fp, "rdcu_model_adr = 0x%06X\n", cfg->rdcu_model_adr); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# RDCU updated model start address, the address in the RDCU SRAM where the updated model is stored\n"); - printf("\n"); - printf("rdcu_new_model_adr = 0x%06X\n", + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# RDCU updated model start address, the address in the RDCU SRAM where the updated model is stored\n"); + fprintf(fp, "\n"); + fprintf(fp, "rdcu_new_model_adr = 0x%06X\n", cfg->rdcu_new_model_adr); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); - printf("# RDCU compressed data start address, the first output data address in the RDCU SRAM\n"); - printf("\n"); - printf("rdcu_buffer_adr = 0x%06X\n", cfg->rdcu_buffer_adr); - printf("\n"); - printf("#-------------------------------------------------------------------------------\n"); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); + fprintf(fp, "# RDCU compressed data start address, the first output data address in the RDCU SRAM\n"); + fprintf(fp, "\n"); + fprintf(fp, "rdcu_buffer_adr = 0x%06X\n", cfg->rdcu_buffer_adr); + fprintf(fp, "\n"); + fprintf(fp, "#-------------------------------------------------------------------------------\n"); } } +/** + * @brief prints config struct + * + * @param cfg configuration to print + * @param rdcu_cfg if set additional RDCU parameter are printed as well + */ + +void print_cfg(const struct cmp_cfg *cfg, int rdcu_cfg) +{ + write_cfg_internal(stdout, cfg, rdcu_cfg); +} + +/** + * @brief write compression configuration to a file + * + * @param cfg configuration to print + * @param output_prefix prefix of the written file (.cfg is added to the prefix) + * @param rdcu_cfg if non-zero additional RDCU parameter are printed as well + * @param verbose print verbose output if not zero + * + * @returns 0 on success, error otherwise + */ + +int write_cfg(const struct cmp_cfg *cfg, const char *output_prefix, int rdcu_cfg, + int verbose) +{ + FILE *fp = open_file(output_prefix, ".cfg"); + + if (fp == NULL) { + fprintf(stderr, "%s: %s%s: %s\n", PROGRAM_NAME, output_prefix, + ".cfg", strerror(errno)); + return -1; + } + + write_cfg_internal(fp, cfg, rdcu_cfg); + + fclose(fp); + + if (verbose) + print_cfg(cfg, rdcu_cfg); + return 0; +} + + /** * @brief write a decompression information structure to a file * * @param info compressor information contains information of an executed * compression * @param output_prefix prefix of the written file (.info is added to the prefix) - * @param machine_cfg - if set write additional RDCU parameter in the file + * @param rdcu_cfg - if non-zero write additional RDCU parameter in the file * * @returns 0 on success, error otherwise */ int write_info(const struct cmp_info *info, const char *output_prefix, - int machine_cfg) + int rdcu_cfg) { FILE *fp = open_file(output_prefix, ".info"); @@ -1200,7 +1295,7 @@ int write_info(const struct cmp_info *info, const char *output_prefix, fprintf(fp, "\n"); fprintf(fp, "#-------------------------------------------------------------------------------\n"); - if (machine_cfg) { + if (rdcu_cfg) { fprintf(fp, "#-------------------------------------------------------------------------------\n"); fprintf(fp, "# Hardware Compressor Settings (not need for SW compression)\n"); fprintf(fp, "#-------------------------------------------------------------------------------\n");