diff --git a/.gitignore b/.gitignore
index 3158f7eeea28e78922541cd9d8225765aa61719b..b73bda620d62a0e9e4bc0c7ffd089a3a32bbed1f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
### C ###
# Executables
cmp_tool
+!test/cmp_tool
# Prerequisites
*.d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fdbf87185732d1d4d5ba817c527ce231a3333131..f74a06c7db46a58d1de9780de5e0f7722cc58da0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,13 @@
All notable changes to this project will be documented in this file.
## [Unreleased]
+## [0.06] - 12-11-2021
+- if samples = 0 the cmp_tool counts the samples in the data file and uses them
+- if buffer_length = 0 the 2*samples parameter is used as buffer_length
+- added feature to guess the compression configuration
+- some code restructuring
+- added more detailed error messages
+- small bug fixes
## [0.05] - 23-04-2021
### Removed
- discard old file format. Now the only input format is like: 12 AB 23 CD .. ..
diff --git a/Makefile b/Makefile
index d617c783dfc46652e014b622a76c531026dcacea..28c93b0de9d9d273e1bd1656a0996b0b389bbd14 100644
--- a/Makefile
+++ b/Makefile
@@ -2,9 +2,11 @@ CC = gcc
SOURCEDIR = lib
INCLUDEDIR = include
BUILDDIR = ./
-CFLAGS := -Wall -Wextra -std=gnu99 -pedantic -Wshadow
+CFLAGS := -Wall -Wextra -std=gnu99 -pedantic -Wshadow \
+ -Wunreachable-code #-Wdocumentation #-Wsign-conversion
RELCFLAGS := -O2 # Release flags
DBCFLAGS := -O0 -g3 #debug flags
+COVFLAGS := -fprofile-arcs -ftest-coverage #coverage flags
CPPFLAGS := -I $(INCLUDEDIR)
LDFLAGS := -lm
SOURCES := cmp_tool.c \
@@ -18,7 +20,8 @@ SOURCES := cmp_tool.c \
$(SOURCEDIR)/cmp_icu.c \
$(SOURCEDIR)/decmp.c \
$(SOURCEDIR)/rdcu_pkt_to_file.c \
- $(SOURCEDIR)/tool_lib.c
+ $(SOURCEDIR)/cmp_guess.c \
+ $(SOURCEDIR)/cmp_tool_lib.c
TARGET := cmp_tool
DEBUG?=1
@@ -34,3 +37,13 @@ all: $(SOURCES)
debug: $(SOURCES)
$(CC) $(CPPFLAGS) $(CFLAGS) $(DBCFLAGS) $^ -o $(TARGET) $(LDFLAGS)
+
+integration: $(SOURCES)
+ $(CC) $(CPPFLAGS) $(CFLAGS) $(DBCFLAGS) $(COVFLAGS) $^ -o $(TARGET) $(LDFLAGS)
+ pytest test/cmp_tool/cmp_tool_integration_test.py
+ lcov --rc lcov_branch_coverage=1 --capture --directory ./ --output-file coverage.info
+ genhtml --rc lcov_branch_coverage=1 --branch-coverage coverage.info --output-directory out
+#firefox out/index.html
+
+clean:
+ rm -rf $(TARGET) *.gcno *.gcda coverage.info out
diff --git a/README.md b/README.md
index 440ec46bc5b22ef158c1ed69886ab0f6f0829724..66378e4532ab428e264fa94f5d54744bb6e66d51 100644
--- a/README.md
+++ b/README.md
@@ -1,100 +1,139 @@
# PLATO Compression/Decompression Tool
-If you find a bug or have a feature request please file an [issue][1] or send me an [email][2]
-Compiled executable can be found [here](https://gitlab.phaidra.org/loidoltd15/cmp_tool/-/tags).
+If you find a bug or have a feature request please file an [issue][1] or send
+me an [email][2].
+Compiled executables can be found [here][3].
## Usage
-**usage:** `./cmp_tool [options] [<argument>]`
-### General Options
-| Options | Description |
-| ----------------- | ---------------------------------------------------------------------------------- |
-| `-h, --help` | Print this help text and exit |
-| `-V, --version` | Print program version and exit |
-| `-v, --verbose` | Print various debugging information |
-| `-n, --model_cfg` | Print a default model configuration and exit<sup>[1](#myfootnote2)</sup> |
-| `--diff_cfg` | Print a default 1d-differencing configuration and exit<sup>[1](#myfootnote2)</sup> |
-| `-a, --rdcu_par` | Print additional RDCU control parameters |
-| `-o <prefix>` | Use the `<prefix>` for output files<sup>[2](#myfootnote2)</sup> |
-
-<a name="myfootnote1">1</a>) **NOTE:** In the default configurations the samples and buffer_length parameter is set to 0!
-<a name="myfootnote2">2</a>) **NOTE:** If the -o option is not used the `<prefix>` will be set to "OUTPUT".
-
-### Compression Options
-| Options | Description |
-| --------------------------- | ----------------------------------------------------------------------------------- |
-| `-c <file>` | File containing the compressing configuration |
-| `-d <file>` | File containing the data to be compressed |
-| `-m <file>` | File containing the model of the data to be compressed |
-| `--rdcu_pkt` | Generate RMAP packets for a RDCU compression<sup>[3](#myfootnote3)</sup> |
-| `--last_info <.info file>` | Generate RMAP packets for a RDCU compression with parallel read of the last results |
-
-<a name="myfootnote2">3</a>) **NOTE:** When using the `--rdcu_pkt` option the configuration of the RMAP parameters
- can be found in the `.rdcu_pkt_mode_cfg file`. The generated packets can be
- found in the `TC_FILES` directory.
-
-### Decompression Options
+
+**usage:** `cmp_tool [options] [<argument>]`
+
+### General Options
+
+| Options | Description |
+|:------------------|:------------------------------------------------------------------------------|
+| `-h, --help` | Print some help text and exit |
+| `-V, --version` | Print program version and exit |
+| `-v, --verbose` | Print various debugging information |
+| `-n, --model_cfg` | Print a default model configuration and exit<sup>[1](#fnote1)</sup> |
+| `--diff_cfg` | Print a default 1d-differencing configuration and exit<sup>[1](#fnote1)</sup> |
+| `-a, --rdcu_par` | Add additional RDCU control parameters |
+| `-o <prefix>` | Use the `<prefix>` for output files<sup>[2](#fnote2)</sup> |
+
+<a name="fnote1">1</a>) **NOTE:** In the default configurations the **samples**
+and **buffer_length** parameter is set to **0**!
+<a name="fnote2">2</a>) **NOTE:** If the -o option is not used the `<prefix>`
+will be set to "OUTPUT".
+
+### Compression Options
+
+| Options | Description |
+|:----------------------------|:-------------------------------------------------------------------------------------|
+| `-c <file>` | File containing the compressing configuration |
+| `-d <file>` | File containing the data to be compressed |
+| `-m <file>` | File containing the model of the data to be compressed |
+| `--rdcu_pkt` | Generate RMAP packets for an RDCU compression<sup>[3](#fnoot3)</sup> |
+| `--last_info <.info file>` | Generate RMAP packets for an RDCU compression with parallel read of the last results |
+
+<a name="foot3">3</a>) **NOTE:** When using the `--rdcu_pkt` option the
+configuration of the RMAP parameters can be found in the `.rdcu_pkt_mode_cfg file`.
+The generated packets can be found in the `TC_FILES` directory.
+
+### Decompression Options
+
| Options | Description |
-| ----------- | ------------------------------------------------ |
+|:----------- |:-------------------------------------------------|
| `-i <file>` | File containing the decompression information |
-| `-d <file>` | File containing the compressed data |
-| `-m <file>` | File containing the model of the compressed data |
+| `-d <file>` | File containing the compressed data |
+| `-m <file>` | File containing the model of the compressed data |
+
+### Guessing Options
+
+| Options | Description |
+|:------------------------|:--------------------------------------------------------------------------------|
+| `--guess <mode>` | Search for a good configuration for compression \<mode\><sup>[4](#fnoot4)</sup> |
+| `-d <file>` | File containing the data to be compressed |
+| `-m <file>` | File containing the model of the data to be compressed |
+| `--guess_level <level>` | Set guess level to \<level\> (optional)<sup>[5](#fnoot5)</sup> |
+
+<a name="fnoot4">4</a>) **NOTE:** \<mode\> can be either the compression mode
+number or the keyword: `RDCU`. The RDCU mode automatically selects the correct
+RDCU-compatible compression mode depending on if the Model (-m) option is set.
+<a name="fnoot5">5</a>) **Supported levels:**
-### Data Formart
-The input data must be formatted as two hex numbers separated by a space.
+| guess level | Description |
+|:------------|:--------------------------------|
+| `1` | fast mode (not implemented yet) |
+| `2` | default mode |
+| `3` | brute force |
+
+**Example of Compression Parmeter guessing:**
+
+``./cmp_tool --guess RDCU -d test_data/test_data1.dat -o myguess``
+
+This command creates the file `myguess.cfg` with the guessed compression parameters.
+
+### Data Format
+
+The input data must be formatted as two hex numbers separated by a space.
For example: `12 AB 34 CD`.
## How to use the tool
+
A simple example to show how the compression tool works.
-0. Run `make` to build the tool
+0. Download the [tool][3] or run `make` to build the tool
-1. Create a configuration file
-* Create a cfg directory
+1. Create a configuration file
+* Create a cfg directory
`mkdir cfg`
-* To create a default 1d-differencing mode configuration:
+* To create a default 1d-differencing mode configuration:
`./cmp_tool --diff_cfg > cfg/default_config_1d.cfg`
-* To create a default model mode configuration:
+* To create a default model mode configuration:
`./cmp_tool --model_cfg > cfg/default_config_model.cfg`
-* Change the the `samples` and `buffer_length` parameters from `0` to `50` in the `default_config_1d.cfg` and `default_config_model.cfg` files
+* Change the the **`samples`** and **`buffer_length`** parameters from `0` to `50`
+in the `default_config_1d.cfg` and `default_config_model.cfg` files
-2. Compress data with the default configurations:
-* Create a directory for the compressed data.
- `mkdir compressed`
-* 1d-differencing mode compression
+2. Compress data with the default configurations:
+* Create a directory for the compressed data
+ `mkdir compressed`
+* 1d-differencing mode compression
`./cmp_tool -c cfg/default_config_1d.cfg -d test_data/test_data1.dat -o compressed/data1`
-
- This creates this two files in compressed directory:
- `data1.cmp ` -> compressed `test_data1.dat` data
- `data1.info` -> decompression information for `data1.cmp`
-
-* Model mode compression
+ This creates these two files in the compressed directory:
+ `data1.cmp` -> compressed `test_data1.dat` file
+ `data1.info` -> decompression information for `data1.cmp`
+* Model mode compression
`./cmp_tool -c cfg/default_config_model.cfg -d test_data/test_data2.dat -m test_data/test_data1.dat -o compressed/data2`
- We use `test_data1.dat`as the first model for `test_data2.dat`
+ We use `test_data1.dat` as the first model for `test_data2.dat`.
- Creates this three files in compressed directory:
- `data2.cmp ` -> compressed `test_data3.dat` data
+ Creates these three files in the compressed directory:
+ `data2.cmp ` -> compressed `test_data3.dat` file
`data2.info` -> decompression information for `data2.cmp`
- `data2_upmodel.dat` -> updated model used to **compress** the next data in model mode
-
-3. Decompress the data
-* Create a directory for the decompressed data.
- `mkdir decompressed`
+ `data2_upmodel.dat` -> updated model used to **compress** the next data in model mode
-* Decompress `data1.cmp`
+3. Decompress the data
+* Create a directory for the decompressed data
+ `mkdir decompressed`
+* Decompress `data1.cmp`
`./cmp_tool -i compressed/data1.info -d compressed/data1.cmp -o decompressed/test_data1`
-
-* Decompress `data2.cmp`
+ Creates this file in the decompressed directory:
+ `test_data1.dat ` -> decompressed `data1.cmp` file
+* Decompress `data2.cmp`
`./cmp_tool -i compressed/data2.info -d compressed/data2.cmp -m decompressed/test_data1.dat -o decompressed/test_data2`
As for the compression we use `test_data1.dat` as our model for decompression.
+ Creates these two files in the decompressed directory:
+ `test_data2.dat ` -> decompressed `data2.cmp` file
+ `data2_upmodel.dat` -> updated model used to **decompress** the next data in model mode
+
4. Bonus: Check if the decompressed data are equal to the original data
The moment of truth!
`diff test_data/test_data1.dat decompressed/test_data1.dat`
- `diff test_data/test_data2.dat decompressed/test_data2.dat`
+ `diff test_data/test_data2.dat decompressed/test_data2.dat`
And also check if the updated model is the same
- `diff compressed/data2_upmodel.dat decompressed/test_data2_upmodel.dat`
-
+ `diff compressed/data2_upmodel.dat decompressed/test_data2_upmodel.dat`
[1]: <https://gitlab.phaidra.org/loidoltd15/cmp_tool/-/issues> "issues"
[2]: <mailto:dominik.loidolt@univie.ac.at?subject=%5BIssue%5D%20Cmd_Tool> "email"
+[3]: <https://gitlab.phaidra.org/loidoltd15/cmp_tool/-/releases> "release"
\ No newline at end of file
diff --git a/cmp_tool.c b/cmp_tool.c
index 90ce973e647312a0a279e3cfce7851e3d7618055..a314b8f5b3b844718a4adbd4a67c1fa0a8d7cfbe 100755
--- a/cmp_tool.c
+++ b/cmp_tool.c
@@ -19,28 +19,37 @@
* @see Data Compression User Manual PLATO-UVIE-PL-UM-0001
*/
+#include <stdio.h>
+#include <stdlib.h>
#include <limits.h>
#include <getopt.h>
-#include "include/tool_lib.h"
+#include "include/cmp_tool_lib.h"
#include "include/cmp_icu.h"
-#include "include/cmp_rdcu.h"
#include "include/decmp.h"
+#include "include/cmp_guess.h"
+#include "include/rdcu_pkt_to_file.h"
+
+
+#define VERSION "0.06"
+
+#define BUFFER_LENGTH_DEF_FAKTOR 2
-#define VERSION "0.05"
/*
* For long options that have no equivalent short option, use a
* non-character as a pseudo short option, starting with CHAR_MAX + 1.
*/
enum {
- RDCU_PKT_OPTION = CHAR_MAX + 1,
- DIFF_CFG_OPTION,
- RDCU_PAR_OPTION,
+ DIFF_CFG_OPTION = CHAR_MAX + 1,
+ GUESS_OPTION,
+ GUESS_LEVEL,
+ RDCU_PKT_OPTION,
LAST_INFO,
+
};
-static struct option const long_options[] = {
+static const struct option long_options[] = {
{"rdcu_par", no_argument, NULL, 'a'},
{"model_cfg", no_argument, NULL, 'n'},
{"help", no_argument, NULL, 'h'},
@@ -48,9 +57,41 @@ static struct option const long_options[] = {
{"version", no_argument, NULL, 'V'},
{"rdcu_pkt", no_argument, NULL, RDCU_PKT_OPTION},
{"diff_cfg", no_argument, NULL, DIFF_CFG_OPTION},
- {"last_info", required_argument, NULL, LAST_INFO}
+ {"guess", required_argument, NULL, GUESS_OPTION},
+ {"guess_level", required_argument, NULL, GUESS_LEVEL},
+ {"last_info", required_argument, NULL, LAST_INFO},
+ {NULL, 0, NULL, 0}
};
+/* prefix of the generated output file names */
+static const char *output_prefix = DEFAULT_OUTPUT_PREFIX;
+
+/* if non zero additional RDCU parameters are included in the compression
+ * configuration and decompression information files */
+static int print_rdcu_cfg;
+
+/* if non zero generate RDCU setup packets */
+static int rdcu_pkt_mode;
+
+/* file name of the last compression information file to generate parallel RDCU
+ * setup packets */
+static const char *last_info_file_name;
+
+/* if non zero print additional verbose output */
+static int verbose_en;
+
+
+/* find a good set of compression parameters for a given dataset */
+static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode,
+ int guess_level);
+
+/* compress the data and write the results to files */
+static int compression(struct cmp_cfg *cfg, struct cmp_info *info);
+
+/* decompress the data and write the results in file(s)*/
+static int decompression(uint32_t *decomp_input_buf, uint16_t *input_model_buf,
+ struct cmp_info *info);
+
/**
* @brief This is the main function of the compression / decompression tool
@@ -64,38 +105,37 @@ static struct option const long_options[] = {
int main(int argc, char **argv)
{
- int32_t opt;
+ int opt;
+ int error;
const char *cfg_file_name = NULL;
const char *info_file_name = NULL;
const char *data_file_name = NULL;
const char *model_file_name = NULL;
- const char *last_info_file_name = NULL;
- const char *output_prefix = DEFAULT_OUTPUT_PREFIX;
+ const char *guess_cmp_mode = NULL;
+ const char *program_name = argv[0];
+ int cmp_operation = 0;
int print_model_cfg = 0;
+ int guess_operation = 0;
+ int guess_level = DEFAULT_GUESS_LEVEL;
int print_diff_cfg = 0;
- int print_rdcu_cfg = 1;
- int cmp_operation = 0;
- int verbose_en = 0;
- int rdcu_pkt_mode = 0;
struct cmp_cfg cfg = {0}; /* compressor configuration struct */
struct cmp_info info = {0}; /* decompression information struct */
- struct cmp_info last_info = {0}; /* last decompression information struct */
- int error;
uint32_t *decomp_input_buf = NULL;
uint16_t *input_model_buf = NULL;
+ ssize_t size;
/* show help if no arguments are provided */
if (argc < 2) {
- Print_Help(*argv);
+ print_help(program_name);
exit(EXIT_FAILURE);
}
- while ((opt = getopt_long (argc, argv, "ac:d:hi:m:no:vV", long_options,
- NULL)) != -1) {
+ while ((opt = getopt_long(argc, argv, "ac:d:hi:m:no:vV", long_options,
+ NULL)) != -1) {
switch (opt) {
case 'a': /* --rdcu_par */
print_rdcu_cfg = 1;
@@ -108,7 +148,7 @@ int main(int argc, char **argv)
data_file_name = optarg;
break;
case 'h': /* --help */
- Print_Help(*argv);
+ print_help(argv[0]);
exit(EXIT_SUCCESS);
break;
case 'i':
@@ -127,12 +167,19 @@ int main(int argc, char **argv)
verbose_en = 1;
break;
case 'V': /* --version */
- printf("%s %s\n", PROGRAM_NAME, VERSION);
+ printf("%s version %s\n", PROGRAM_NAME, VERSION);
exit(EXIT_SUCCESS);
break;
case DIFF_CFG_OPTION:
print_diff_cfg = 1;
break;
+ case GUESS_OPTION:
+ guess_operation = 1;
+ guess_cmp_mode = optarg;
+ break;
+ case GUESS_LEVEL:
+ guess_level = atoi(optarg);
+ break;
case RDCU_PKT_OPTION:
rdcu_pkt_mode = 1;
break;
@@ -141,11 +188,39 @@ int main(int argc, char **argv)
last_info_file_name = optarg;
break;
default:
- Print_Help(*argv);
+ print_help(program_name);
exit(EXIT_FAILURE);
break;
}
}
+ argc -= optind;
+ argv += optind;
+
+ if (argc > 0) {
+ printf("%s: To many arguments.\n", PROGRAM_NAME);
+ print_help(argv[0]);
+ exit(EXIT_FAILURE);
+ }
+#if 0
+ if (argc > 0) {
+ if(!data_file_name)
+ data_file_name = argv[0];
+ else {
+ printf("You can define the data file using either the -d option or the first argument, but not both.\n");
+ print_help(program_name);
+ exit(EXIT_FAILURE);
+ }
+ }
+ if (argc > 1) {
+ if(!model_file_name)
+ model_file_name = argv[1];
+ else {
+ printf("You can define the model file using either the -m option or the second argument, but not both.\n");
+ print_help(program_name);
+ exit(EXIT_FAILURE);
+ }
+ }
+#endif
if (print_model_cfg == 1) {
print_cfg(&DEFAULT_CFG_MODEL, print_rdcu_cfg);
@@ -162,275 +237,370 @@ int main(int argc, char **argv)
VERSION);
printf("#########################################################\n\n");
- if (!cfg_file_name && !info_file_name) {
- fprintf(stderr, "%s: No configuration file (-c option) or decompression information file (-i option) specified.\n",
- PROGRAM_NAME);
- exit(EXIT_FAILURE);
- }
-
if (!data_file_name) {
fprintf(stderr, "%s: No data file (-d option) specified.\n",
PROGRAM_NAME);
exit(EXIT_FAILURE);
}
- /* read input data and .cfg or .info */
- if (cmp_operation) { /* compression mode */
- printf("### Compression ###\n");
- printf("Importing configuration file %s ... ", cfg_file_name);
+ if (!cfg_file_name && !info_file_name && !guess_operation) {
+ fprintf(stderr, "%s: No configuration file (-c option) or decompression information file (-i option) specified.\n",
+ PROGRAM_NAME);
+ exit(EXIT_FAILURE);
+ }
- error = read_cmp_cfg(cfg_file_name, &cfg, verbose_en);
- if (error) {
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
- printf("DONE\n");
+ /* read input data and .cfg or .info */
+ if (cmp_operation || guess_operation) { /* compression mode */
+ if (cmp_operation) {
+ /* printf("### Compression ###\n"); */
+ printf("Importing configuration file %s ... ", cfg_file_name);
+ error = read_cmp_cfg(cfg_file_name, &cfg, verbose_en);
+ if (error)
+ goto fail;
+ else
+ printf("DONE\n");
+ }
printf("Importing data file %s ... ", data_file_name);
- cfg.input_buf = read_file16(data_file_name, cfg.samples,
- verbose_en);
+ /* count the samples in the data file when samples == 0 */
+ if (cfg.samples == 0) {
+ size = read_file16(data_file_name, NULL, 0, 0);
+ if (size < 0)
+ goto fail;
+ if (size%size_of_a_sample(cfg.cmp_mode) != 0) {
+ fprintf(stderr, "\n%s: %s: Error: The data file is not correct formatted. Expected multiple of %zu hex words.\n",
+ PROGRAM_NAME, data_file_name, size_of_a_sample(cfg.cmp_mode));
+ goto fail;
+ }
+
+ cfg.samples = size/size_of_a_sample(cfg.cmp_mode);
+ printf("\nNo samples parameter set. Use samples = %u.\n... ",
+ cfg.samples);
+ }
+
+ cfg.input_buf = malloc(cfg.samples * size_of_a_sample(cfg.cmp_mode));
if (!cfg.input_buf) {
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
+ fprintf(stderr, "%s: Error allocating memory for input data buffer.\n", PROGRAM_NAME);
+ goto fail;
+ }
+
+ size = read_file16(data_file_name, cfg.input_buf, cfg.samples,
+ verbose_en);
+ if (size < 0)
+ goto fail;
+ else
printf("DONE\n");
} else { /* decompression mode*/
uint32_t cmp_size_byte;
- printf("### Decompression ###\n\n");
- printf("Importing decompression information file ... ");
+ /* printf("### Decompression ###\n\n"); */
+ printf("Importing decompression information file %s ... ", info_file_name);
error = read_cmp_info(info_file_name, &info, verbose_en);
- if (error) {
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
+ if (error)
+ goto fail;
+ else
printf("DONE\n");
-
printf("Importing compressed data file %s ... ", data_file_name);
-
+ /* cmp_size unit is in bits */
cmp_size_byte = (info.cmp_size + 31) / 8;
-
- decomp_input_buf = read_file32(data_file_name, cmp_size_byte/4,
- verbose_en);
+ decomp_input_buf = malloc(cmp_size_byte);
if (!decomp_input_buf) {
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
+ fprintf(stderr, "%s: Error allocating memory for decompression input buffer.\n", PROGRAM_NAME);
+ goto fail;
+ }
+
+ size = read_file32(data_file_name, decomp_input_buf,
+ cmp_size_byte/4, verbose_en);
+ if (size < 0)
+ goto fail;
+ else
printf("DONE\n");
}
- /* read in model for compressed and decompression */
+ /* read in model */
if ((cmp_operation && model_mode_is_used(cfg.cmp_mode)) ||
- (!cmp_operation && model_mode_is_used(info.cmp_mode_used))) {
+ (!cmp_operation && model_mode_is_used(info.cmp_mode_used)) ||
+ (guess_operation && model_file_name)) {
uint32_t model_length;
- if (cmp_operation)
- model_length = cfg.samples;
- else
- model_length = info.samples_used;
-
if (!model_file_name) {
- fprintf(stderr, "%s: No model file (-m "
- "option) specified.\n", PROGRAM_NAME);
- exit(EXIT_FAILURE);
+ fprintf(stderr, "%s: No model file (-m option) specified.\n", PROGRAM_NAME);
+ goto fail;
}
printf("Importing model file %s ... ", model_file_name);
+ if (cmp_operation || guess_operation)
+ model_length = cfg.samples;
+ else
+ model_length = info.samples_used;
- input_model_buf = read_file16(model_file_name, model_length,
- verbose_en);
+ input_model_buf = malloc(model_length * size_of_a_sample(cfg.cmp_mode));
if (!input_model_buf) {
- printf("FAILED\n");
- free(cfg.input_buf);
- free(decomp_input_buf);
- exit(EXIT_FAILURE);
- } else
+ fprintf(stderr, "%s: Error allocating memory for model buffer.\n", PROGRAM_NAME);
+ goto fail;
+ }
+
+ size = read_file16(model_file_name, input_model_buf, model_length, verbose_en);
+ if (size < 0)
+ goto fail;
+ else
printf("DONE\n");
- if (cmp_operation)
+ if (cmp_operation || guess_operation)
cfg.model_buf = input_model_buf;
}
- if (cmp_operation) { /* perform a compression */
- uint32_t cmp_size_byte;
+ if (guess_operation) {
+ error = guess_cmp_pars(&cfg, guess_cmp_mode, guess_level);
+ if (error)
+ goto fail;
+ } else if (cmp_operation) { /* perform a compression */
+ error = compression(&cfg, &info);
+ if (error)
+ goto fail;
+ } else { /* perform a decompression */
+ error = decompression(decomp_input_buf, input_model_buf, &info);
+ if (error)
+ goto fail;
+ }
- if (rdcu_pkt_mode) {
- printf("Generate compression setup packets ... \n");
- error = init_rmap_pkt_to_file();
- if (error) {
- printf("... FAILED\n");
- fprintf(stderr, "%s: Read RMAP packet config file .rdcu_pkt_mode_cfg failed.\n",
- PROGRAM_NAME);
- free(cfg.input_buf);
- free(input_model_buf);
- exit(EXIT_FAILURE);
- }
+ /* write our the updated model for compressed or decompression */
+ if (!guess_operation &&
+ ((cmp_operation && model_mode_is_used(cfg.cmp_mode)) ||
+ (!cmp_operation && model_mode_is_used(info.cmp_mode_used)))) {
+ printf("Write updated model to file %s_upmodel.dat ... ",
+ output_prefix);
+ error = write_to_file16(input_model_buf, info.samples_used,
+ output_prefix, "_upmodel.dat", verbose_en);
+ if (error)
+ goto fail;
+ else
+ printf("DONE\n");
+ }
- if (last_info_file_name) {
- error = read_cmp_info(last_info_file_name,
- &last_info, verbose_en);
- if (error) {
- printf("... FAILED\n");
- fprintf(stderr, "%s: %s: Importing last decompression information file failed.\n",
- PROGRAM_NAME, last_info_file_name);
- exit(EXIT_FAILURE);
- }
-
- error = gen_rdcu_parallel_pkts(&cfg, &last_info);
- if (error) {
- printf("... FAILED\n");
- free(cfg.input_buf);
- free(input_model_buf);
- exit(EXIT_FAILURE);
- }
- }
+ free(cfg.input_buf);
+ free(decomp_input_buf);
+ free(input_model_buf);
- error = gen_write_rdcu_pkts(&cfg);
+ exit(EXIT_SUCCESS);
- if (error) {
- printf("... FAILED\n");
- free(cfg.input_buf);
- free(input_model_buf);
- exit(EXIT_FAILURE);
- } else
- printf("... DONE\n");
- }
+fail:
+ printf("FAILED\n");
- cfg.icu_output_buf = (uint32_t *)malloc(cfg.buffer_length *
- SAM2BYT);
- if (cfg.icu_output_buf == NULL) {
- printf("%s: Error allocating Memory for Output Buffer."
- "\n", PROGRAM_NAME);
- free(cfg.input_buf);
- free(input_model_buf);
- abort();
- }
+ free(cfg.input_buf);
+ free(decomp_input_buf);
+ free(input_model_buf);
- printf("Compress data ... ");
+ exit(EXIT_FAILURE);
+}
- error = icu_compress_data(&cfg, &info);
- free(cfg.input_buf);
- if (error || info.cmp_err != 0) {
- printf("FAILED\n");
- printf("Compression error %#X\n", info.cmp_err);
- free(input_model_buf);
- free(cfg.icu_output_buf);
- exit(EXIT_FAILURE);
- } else
- printf("DONE\n");
- if (rdcu_pkt_mode) {
- printf("Generate the read results packets ... ");
- error = gen_read_rdcu_pkts(&info);
- if (error) {
- printf("FAILED\n");
- free(input_model_buf);
- free(cfg.icu_output_buf);
- exit(EXIT_FAILURE);
- } else
- printf("DONE\n");
- }
- printf("Write compressed data to file %s.cmp ... ",
- output_prefix);
+/* generate packets to setup a RDCU compression */
+static int gen_rdcu_write_pkts(struct cmp_cfg *cfg)
+{
+ int error;
+
+ error = init_rmap_pkt_to_file();
+ if (error) {
+ fprintf(stderr, "%s: Read RMAP packet config file .rdcu_pkt_mode_cfg failed.\n",
+ PROGRAM_NAME);
+ return -1;
+ }
- /* length of cmp_size in bytes words (round up to 4 bytes) */
- cmp_size_byte = (info.cmp_size + 31)/32 * 4;
+ if (last_info_file_name) {
+ /* generation of packets for parallel read/write RDCU setup */
+ struct cmp_info last_info = {0};
- error = write_cmp_data_file(cfg.icu_output_buf, cmp_size_byte,
- output_prefix, ".cmp", verbose_en);
- free(cfg.icu_output_buf);
+ error = read_cmp_info(last_info_file_name, &last_info, verbose_en);
if (error) {
- printf("FAILED\n");
- free(input_model_buf);
- exit(EXIT_FAILURE);
- } else
- printf("DONE\n");
-
- if (model_mode_is_used(cfg.cmp_mode)) {
- printf("Write updated model to file %s_upmodel.dat ... "
- , output_prefix);
- error = write_to_file16(input_model_buf, info.samples_used,
- output_prefix, "_upmodel.dat",
- verbose_en);
- free(input_model_buf);
- if (error) {
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
- printf("DONE\n");
+ fprintf(stderr, "%s: %s: Importing last decompression information file failed.\n",
+ PROGRAM_NAME, last_info_file_name);
+ return -1;
}
- printf("Write decompression information to file %s.info ... ",
- output_prefix);
+ error = gen_rdcu_parallel_pkts(cfg, &last_info);
+ if (error)
+ return -1;
+ }
- error = write_info(&info, output_prefix, print_rdcu_cfg);
- if (error) {
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
- printf("DONE\n");
+ /* generation of packets for non-parallel read/write RDCU setup */
+ error = gen_write_rdcu_pkts(cfg);
+ if (error)
+ return -1;
- if (verbose_en) {
- printf("\n");
- print_cmp_info(&info);
- printf("\n");
- }
- } else { /* perform a decompression */
- uint16_t *decomp_output;
-
- decomp_output = (uint16_t *)malloc(info.samples_used * SAM2BYT);
- if (decomp_output == NULL) {
- printf("Error allocating Memory for decmpr Model Buffer\n");
- free(decomp_input_buf);
- free(input_model_buf);
- abort();
+ return 0;
+}
+
+
+/* find a good set of compression parameters for a given dataset */
+static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode,
+ int guess_level)
+{
+ int error;
+ uint32_t cmp_size;
+ float cr;
+
+ printf("Search for a good set of compression parameters (level: %d) ... ", guess_level);
+ if (!strcmp(guess_cmp_mode, "RDCU")) {
+ if (cfg->model_buf)
+ cfg->cmp_mode = CMP_GUESS_DEF_MODE_MODEL;
+ else
+ cfg->cmp_mode = CMP_GUESS_DEF_MODE_DIFF;
+ } else {
+ error = cmp_mode_parse(guess_cmp_mode, &cfg->cmp_mode);
+ if (error) {
+ fprintf(stderr, "%s: Error: unknown compression mode: %s\n", PROGRAM_NAME, guess_cmp_mode);
+ return -1;
}
+ }
+ if (model_mode_is_used(cfg->cmp_mode) && !cfg->model_buf) {
+ fprintf(stderr, "%s: Error: model mode needs model data (-m option)\n", PROGRAM_NAME);
+ return -1;
+ }
- printf("Decompress data ... ");
+ cmp_size = cmp_guess(cfg, guess_level);
+ if (!cmp_size)
+ return -1;
+ printf("DONE\n");
- error = decompress_data(decomp_input_buf, input_model_buf,
- &info, decomp_output);
- free(decomp_input_buf);
- if (error != 0) {
- free(decomp_output);
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
- printf("DONE\n");
+ printf("Write the guessed compression configuration to file %s.cfg ... ", output_prefix);
+ error = write_cfg(cfg, output_prefix, print_rdcu_cfg, verbose_en);
+ if (error)
+ return -1;
+ printf("DONE\n");
- printf("Write decompressed data to file %s.dat ... ",
- output_prefix);
+ cr = (8.0 * cfg->samples * size_of_a_sample(cfg->cmp_mode))/cmp_size;
+ printf("Guessed parameters can compress the data with a CR of %.2f.\n", cr);
- error = write_to_file16(decomp_output, info.samples_used,
- output_prefix, ".dat", verbose_en);
- free(decomp_output);
- if (error) {
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
+ return 0;
+}
+
+
+/* compress the data and write the results to files */
+static int compression(struct cmp_cfg *cfg, struct cmp_info *info)
+{
+ int error;
+ uint32_t cmp_size_byte;
+
+ cfg->icu_output_buf = NULL;
+
+ if (cfg->buffer_length == 0) {
+ cfg->buffer_length = (cfg->samples+1) * BUFFER_LENGTH_DEF_FAKTOR; /* +1 to prevent malloc(0)*/
+ printf("No buffer_length parameter set. Use buffer_length = %u as compression buffer size.\n",
+ cfg->buffer_length);
+ }
+
+ if (rdcu_pkt_mode) {
+ printf("Generate compression setup packets ...\n");
+ error = gen_rdcu_write_pkts(cfg);
+ if (error)
+ goto error_cleanup;
+ else
+ printf("... DONE\n");
+ }
+
+ printf("Compress data ... ");
+
+ cfg->icu_output_buf = malloc(cfg->buffer_length * size_of_a_sample(cfg->cmp_mode));
+ if (cfg->icu_output_buf == NULL) {
+ fprintf(stderr, "%s: Error allocating memory for output buffer.\n", PROGRAM_NAME);
+ goto error_cleanup;
+ }
+
+ error = icu_compress_data(cfg, info);
+ if (error || info->cmp_err != 0) {
+ printf("\nCompression error 0x%02X\n... ", info->cmp_err);
+ if ((info->cmp_err >> SMALL_BUFFER_ERR_BIT) & 1U)
+ fprintf(stderr, "%s: the buffer for the compressed data is too small. Try a larger buffer_length parameter.\n", PROGRAM_NAME);
+ goto error_cleanup;
+ } else
+ printf("DONE\n");
+
+ if (rdcu_pkt_mode) {
+ printf("Generate the read results packets ... ");
+ error = gen_read_rdcu_pkts(info);
+ if (error)
+ goto error_cleanup;
+ else
printf("DONE\n");
+ }
- if (model_mode_is_used(info.cmp_mode_used)) {
- printf("Write updated model to file %s_upmodel.dat ... "
- , output_prefix);
- error = write_to_file16(input_model_buf,
- info.samples_used,
- output_prefix, "_upmodel.dat",
- verbose_en);
- free(input_model_buf);
- if (error) {
- printf("FAILED\n");
- exit(EXIT_FAILURE);
- } else
- printf("DONE\n");
- }
+ printf("Write compressed data to file %s.cmp ... ", output_prefix);
+ /* length of cmp_size in bytes words (round up to 4 bytes) */
+ cmp_size_byte = (info->cmp_size + 31)/32 * 4;
+ error = write_cmp_data_file(cfg->icu_output_buf, cmp_size_byte,
+ output_prefix, ".cmp", verbose_en);
+ if (error)
+ goto error_cleanup;
+ else
+ printf("DONE\n");
+ free(cfg->icu_output_buf);
+ cfg->icu_output_buf = NULL;
+
+ printf("Write decompression information to file %s.info ... ",
+ output_prefix);
+ error = write_info(info, output_prefix, print_rdcu_cfg);
+ if (error)
+ goto error_cleanup;
+ else
+ printf("DONE\n");
+
+ if (verbose_en) {
+ printf("\n");
+ print_cmp_info(info);
+ printf("\n");
}
- exit(EXIT_SUCCESS);
+ return 0;
+
+error_cleanup:
+ free(cfg->icu_output_buf);
+ return -1;
+}
+
+
+/* decompress the data and write the results in file(s)*/
+static int decompression(uint32_t *decomp_input_buf, uint16_t *input_model_buf,
+ struct cmp_info *info)
+{
+ int error;
+ uint16_t *decomp_output;
+
+ printf("Decompress data ... ");
+
+ if (info->samples_used == 0)
+ return 0; /* nothing to decompress */
+
+ decomp_output = malloc(info->samples_used *
+ size_of_a_sample(info->cmp_mode_used));
+ if (decomp_output == NULL) {
+ fprintf(stderr, "%s: Error allocating memory for decompressed data.\n", PROGRAM_NAME);
+ return -1;
+ }
+
+ error = decompress_data(decomp_input_buf, input_model_buf, info,
+ decomp_output);
+ if (error) {
+ free(decomp_output);
+ return -1;
+ }
+ printf("DONE\n");
+
+ printf("Write decompressed data to file %s.dat ... ", output_prefix);
+
+ error = write_to_file16(decomp_output, info->samples_used,
+ output_prefix, ".dat", verbose_en);
+ free(decomp_output);
+ if (error)
+ return -1;
+
+ printf("DONE\n");
+
+ return 0;
}
diff --git a/include/cmp_guess.h b/include/cmp_guess.h
new file mode 100644
index 0000000000000000000000000000000000000000..fe2ed0989a6b8ab188c92a33cb0b43a3ba2414e1
--- /dev/null
+++ b/include/cmp_guess.h
@@ -0,0 +1,38 @@
+/**
+ * @file cmp_guess.h
+ * @author Dominik Loidolt (dominik.loidolt@univie.ac.at),
+ * @date 2020
+ *
+ * @copyright GPLv2
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * @brief helps the user to find a good compression parameters for a given
+ * dataset
+ */
+
+#ifndef CMP_GUESS_H
+#define CMP_GUESS_H
+
+#include "cmp_support.h"
+
+#define DEFAULT_GUESS_LEVEL 2
+
+#define CMP_GUESS_DEF_MODE_DIFF MODE_DIFF_ZERO
+#define CMP_GUESS_DEF_MODE_MODEL MODE_MODEL_MULTI
+
+/* how often the model is updated before it is reset default value */
+#define CMP_GUESS_N_MODEL_UPDATE_DEF 8
+
+uint32_t cmp_guess(struct cmp_cfg *cfg, int level);
+void cmp_guess_set_model_updates(int n_model_updates);
+
+uint16_t cmp_guess_model_value(int n_model_updates);
+
+#endif /* CMP_GUESS_H */
diff --git a/include/cmp_icu.h b/include/cmp_icu.h
index a2acd62bef5653127ecd061b2829f4c0eb2d58ab..cd7f193e10ba3e6b89a863c1f259d948f7c0ce8f 100644
--- a/include/cmp_icu.h
+++ b/include/cmp_icu.h
@@ -25,4 +25,9 @@
int icu_compress_data(struct cmp_cfg *cfg, struct cmp_info *info);
+
+int cmp_pre_process(struct cmp_cfg *cfg);
+int cmp_map_to_pos(struct cmp_cfg *cfg);
+uint32_t cmp_encode_data(struct cmp_cfg *cfg);
+
#endif /* _CMP_ICU_H_ */
diff --git a/include/cmp_support.h b/include/cmp_support.h
index 62dd1e6b08c4c3539f3fe0331979640e16fcacfa..aeafd0df6209cc39b14ed8648ad06d69f748fe29 100644
--- a/include/cmp_support.h
+++ b/include/cmp_support.h
@@ -62,6 +62,8 @@
#define SAM2BYT \
2 /* sample to byte conversion factor; one samples has 16 bits (2 bytes) */
+#define CMP_GOOD_SPILL_DIFF_MULTI 2 /* good guess for the spill parameter using the MODE_DIFF_MULTI */
+
/**
* @brief The cmp_cfg structure can contain the complete configuration of the HW as
* well as the SW compressor.
@@ -163,6 +165,7 @@ int model_mode_is_used(unsigned int cmp_mode);
int diff_mode_is_used(unsigned int cmp_mode);
int raw_mode_is_used(unsigned int cmp_mode);
int rdcu_supported_mode_is_used(unsigned int cmp_mode);
+int cmp_mode_available(unsigned int cmp_mode);
int zero_escape_mech_is_used(unsigned int cmp_mode);
int multi_escape_mech_is_used(unsigned int cmp_mode);
@@ -173,6 +176,7 @@ unsigned int cal_up_model(unsigned int data, unsigned int model, unsigned int
model_value);
uint32_t get_max_spill(unsigned int golomb_par, unsigned int cmp_mode);
+uint32_t cmp_get_good_spill(unsigned int golomb_par, unsigned int cmp_mode);
size_t size_of_a_sample(unsigned int cmp_mode);
unsigned int size_of_bitstream(unsigned int cmp_size);
diff --git a/include/tool_lib.h b/include/cmp_tool_lib.h
similarity index 68%
rename from include/tool_lib.h
rename to include/cmp_tool_lib.h
index 3592078d00c03734ef461f0b54544dee30d6e44d..b8eadaadd57c93fdf3f4e661bdf3db575f4bba09 100644
--- a/include/tool_lib.h
+++ b/include/cmp_tool_lib.h
@@ -1,5 +1,5 @@
/**
- * @file tool_lib.h
+ * @file cmp_tool_lib.h
* @author Johannes Seelig (johannes.seelig@univie.ac.at)
* @author Dominik Loidolt (dominik.loidolt@univie.ac.at),
* @date 2020
@@ -15,32 +15,37 @@
* more details.
*/
-#include <stdio.h>
-#include <stdlib.h>
#include <string.h>
#include "cmp_support.h"
-#include "rdcu_pkt_to_file.h"
#define PROGRAM_NAME "cmp_tool"
#define MAX_CONFIG_LINE 256
#define DEFAULT_OUTPUT_PREFIX "OUTPUT"
-void Print_Help(const char *argv);
+#define BUFFER_LENGTH_DEF_FAKTOR 2
+
+void print_help(const char *program_name);
int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en);
int read_cmp_info(const char *file_name, struct cmp_info *info, int verbose_en);
-uint16_t * read_file16(const char *file_name, uint32_t samples, int verbose_en);
-uint32_t * read_file32(const char *file_name, uint32_t buf_len, int verbose_en);
+ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word,
+ int verbose_en);
+ssize_t read_file16(const char *file_name, uint16_t *buf, uint32_t samples,
+ int verbose_en);
+ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t samples,
+ int verbose_en);
int write_cmp_data_file(const void *buf, uint32_t buf_size, const char
*output_prefix, const char *name_extension, int verbose);
int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char
*output_prefix, const char *name_extension, int verbose);
-
+int write_info(const struct cmp_info *info, const char *output_prefix,
+ int rdcu_cfg);
+int write_cfg(const struct cmp_cfg *cfg, const char *output_prefix, int rdcu_cfg,
+ int verbose);
void print_cfg(const struct cmp_cfg *cfg, int rdcu_cfg);
-int write_info(const struct cmp_info *info, const char *output_prefix,
- int machine_cfg);
+uint32_t cmp_mode_parse(const char *cmp_mode_str, uint32_t *cmp_mode);
diff --git a/lib/cmp_data_types.c b/lib/cmp_data_types.c
index e6bbe92215cb85640123e9842d062f55666130e4..1df363cb274aa698815432ef3e9ed51557c88d5b 100644
--- a/lib/cmp_data_types.c
+++ b/lib/cmp_data_types.c
@@ -17,9 +17,6 @@
*/
-#include <stddef.h>
-#include <stdio.h>
-
#include "../include/cmp_data_types.h"
#include "../include/cmp_support.h"
#include "../include/cmp_debug.h"
@@ -88,7 +85,7 @@ int de_lossy_rounding_16(uint16_t *data_buf, uint32_t samples_used, uint32_t
for (i = 0; i < samples_used; i++) {
/* check if data are not to big for a overflow */
- uint16_t mask = (uint16_t)(~0 << (16-round_used));
+ uint16_t mask = (uint16_t)(~0U << (16-round_used));
if (data_buf[i] & mask) {
debug_print("de_lossy_rounding_16 failed!\n");
return -1;
@@ -152,7 +149,7 @@ int de_lossy_rounding_32(uint32_t *data_buf, uint32_t samples_used, uint32_t
for (i = 0; i < samples_used; i++) {
/* check if data are not to big for a overflow */
- uint32_t mask = (uint32_t)(~0 << (32-round_used));
+ uint32_t mask = (uint32_t)(~0U << (32-round_used));
if (data_buf[i] & mask) {
debug_print("de_lossy_rounding_32 failed!\n");
return -1;
@@ -295,7 +292,7 @@ struct S_FX_EFX add_S_FX_EFX(struct S_FX_EFX a, struct S_FX_EFX b)
* @note change the data buffer in-place
* @note the exposure_flags are not rounded
*
- * @param data_buf a S_FX_EFX formatted data buffer
+ * @param data S_FX_EFX formatted data buffer
* @param samples the size of the data buffer measured in S_FX_EFX samples
* @param round number of bits to round; if zero no rounding takes place
*
@@ -318,8 +315,7 @@ int lossy_rounding_S_FX_EFX(struct S_FX_EFX *data, unsigned int samples,
if (round == 0)
return 0;
- for (i = 0; i < samples; i++)
- {
+ for (i = 0; i < samples; i++) {
data[i].FX = round_fwd(data[i].FX, round);
data[i].EFX = round_fwd(data[i].EFX, round);
}
@@ -431,8 +427,7 @@ int lossy_rounding_S_FX_NCOB(struct S_FX_NCOB *data_buf, unsigned int samples,
if (round == 0)
return 0;
- for (i = 0; i < samples; i++)
- {
+ for (i = 0; i < samples; i++) {
data_buf[i].FX = round_fwd(data_buf[i].FX, round);
data_buf[i].NCOB_X = round_fwd(data_buf[i].NCOB_X, round);
data_buf[i].NCOB_Y = round_fwd(data_buf[i].NCOB_Y, round);
@@ -559,8 +554,7 @@ int lossy_rounding_S_FX_EFX_NCOB_ECOB(struct S_FX_EFX_NCOB_ECOB *data_buf,
if (round == 0) /* round 0 means loss less compression, no further processing is necessary */
return 0;
- for (i = 0; i < samples; i++)
- {
+ for (i = 0; i < samples; i++) {
data_buf[i].FX = round_fwd(data_buf[i].FX, round);
data_buf[i].NCOB_X = round_fwd(data_buf[i].NCOB_X, round);
data_buf[i].NCOB_Y = round_fwd(data_buf[i].NCOB_Y, round);
@@ -587,8 +581,7 @@ int de_lossy_rounding_S_FX_EFX_NCOB_ECOB(struct S_FX_EFX_NCOB_ECOB *data_buf,
if (round_used == 0) /* round 0 means loss less compression, no further processing is necessary */
return 0;
- for (i = 0; i < samples_used; i++)
- {
+ for (i = 0; i < samples_used; i++) {
uint32_t mask = (~0U << (32-round_used));
if (data_buf[i].FX & mask) {
diff --git a/lib/cmp_guess.c b/lib/cmp_guess.c
new file mode 100644
index 0000000000000000000000000000000000000000..265b9c58ad6fe6eca4111f2feffaa177e49b03fe
--- /dev/null
+++ b/lib/cmp_guess.c
@@ -0,0 +1,319 @@
+/**
+ * @file cmp_guess.c
+ * @author Dominik Loidolt (dominik.loidolt@univie.ac.at),
+ * @date 2021
+ *
+ * @copyright GPLv2
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * @brief helps the user to find a good compression parameters for a given
+ * dataset
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "../include/cmp_icu.h"
+#include "../include/cmp_guess.h"
+
+
+/* how often the model is updated before it is rested */
+static int num_model_updates = CMP_GUESS_N_MODEL_UPDATE_DEF;
+
+
+/**
+ * @brief sets how often the model is updated before model reset for the
+ * cmp_guess function
+ * @note the default value is CMP_GUESS_N_MODEL_UPDATE_DEF
+ * @note this is needed to guess a good model_value
+ *
+ * @param n_model_updates number of model updates
+ */
+
+void cmp_guess_set_model_updates(int n_model_updates)
+{
+ num_model_updates = n_model_updates;
+}
+
+
+/**
+ * @brief guess a good model value
+ *
+ * @param n_model_updates number of model updates
+ *
+ * @returns guessed model_value
+ */
+
+uint16_t cmp_guess_model_value(int n_model_updates)
+{
+ if (n_model_updates <= 2)
+ return 8;
+ if (n_model_updates <= 5)
+ return 10;
+ if (n_model_updates <= 11)
+ return 11;
+ if (n_model_updates <= 21)
+ return 12;
+
+ return 13;
+}
+
+
+/**
+ * @brief guess a good configuration with pre_cal_method
+ *
+ * @param cfg compression configuration structure
+ *
+ * @returns the size in bits of the compressed data of the guessed
+ * configuration; 0 on error
+ */
+
+static uint32_t pre_cal_method(struct cmp_cfg *cfg)
+{
+ uint32_t g;
+ uint32_t cmp_size;
+ uint32_t cmp_size_best = ~0U;
+ uint32_t golomb_par_best = 0;
+ uint32_t spill_best = 0;
+
+ for (g = MIN_RDCU_GOLOMB_PAR; g < MAX_RDCU_GOLOMB_PAR; g++) {
+ uint32_t s = cmp_get_good_spill(g, cfg->cmp_mode);
+
+ cfg->golomb_par = g;
+ cfg->spill = s;
+ cmp_size = cmp_encode_data(cfg);
+ if (cmp_size == 0) {
+ return 0;
+ } else if (cmp_size < cmp_size_best) {
+ cmp_size_best = cmp_size;
+ golomb_par_best = g;
+ spill_best = s;
+ }
+ }
+ cfg->golomb_par = golomb_par_best;
+ cfg->spill = spill_best;
+
+ return cmp_size_best;
+}
+
+
+/**
+ * @brief guess a good configuration with brute force method
+ *
+ * @param cfg compression configuration structure
+ *
+ * @returns the size in bits of the compressed data of the guessed
+ * configuration; 0 on error
+ */
+
+#define CMP_GUESS_MAX_CAL_STEPS 20274
+static uint32_t brute_force(struct cmp_cfg *cfg)
+{
+ uint32_t g, s;
+ uint32_t n_cal_steps = 0, last = 0;
+ const uint32_t max_cal_steps = CMP_GUESS_MAX_CAL_STEPS;
+ uint32_t cmp_size;
+ uint32_t cmp_size_best = ~0U;
+ uint32_t golomb_par_best = 0;
+ uint32_t spill_best = 0;
+ uint32_t percent;
+
+ /* short cut for zero escape mechanism */
+ if (zero_escape_mech_is_used(cfg->cmp_mode))
+ return pre_cal_method(cfg);
+
+ printf("0%%... ");
+ fflush(stdout);
+
+ for (g = MIN_RDCU_GOLOMB_PAR; g < MAX_RDCU_GOLOMB_PAR; g++) {
+ for (s = MIN_RDCU_SPILL; s < get_max_spill(g, cfg->cmp_mode); s++) {
+ cfg->golomb_par = g;
+ cfg->spill = s;
+
+ cmp_size = cmp_encode_data(cfg);
+ if (cmp_size == 0) {
+ return 0;
+ } else if (cmp_size < cmp_size_best) {
+ cmp_size_best = cmp_size;
+ golomb_par_best = g;
+ spill_best = s;
+ }
+ }
+ n_cal_steps += s;
+
+ percent = n_cal_steps*100/max_cal_steps;
+ if (percent > 5+last && percent < 100) {
+ last = percent;
+ printf("%u%%... ", percent);
+ fflush(stdout);
+ }
+ }
+ printf("100%% ");
+ cfg->golomb_par = golomb_par_best;
+ cfg->spill = spill_best;
+
+ return cmp_size_best;
+}
+
+
+/**
+ * @brief guessed rdcu specific adaptive parameters
+ *
+ * @param cfg compression configuration structure
+ * @note internal use only
+ */
+
+static void add_rdcu_pars_internal(struct cmp_cfg *cfg)
+{
+ if (cfg->golomb_par == MIN_RDCU_GOLOMB_PAR) {
+ cfg->ap1_golomb_par = cfg->golomb_par + 1;
+ cfg->ap2_golomb_par = cfg->golomb_par + 2;
+
+ } else if (cfg->golomb_par == MAX_RDCU_GOLOMB_PAR) {
+ cfg->ap1_golomb_par = cfg->golomb_par - 2;
+ cfg->ap2_golomb_par = cfg->golomb_par - 1;
+ } else {
+ cfg->ap1_golomb_par = cfg->golomb_par - 1;
+ cfg->ap2_golomb_par = cfg->golomb_par + 1;
+ }
+
+ cfg->ap1_spill = cmp_get_good_spill(cfg->ap1_golomb_par, cfg->cmp_mode);
+ cfg->ap2_spill = cmp_get_good_spill(cfg->ap2_golomb_par, cfg->cmp_mode);
+
+ if (model_mode_is_used(cfg->cmp_mode)) {
+ cfg->rdcu_data_adr = DEFAULT_CFG_MODEL.rdcu_data_adr;
+ cfg->rdcu_model_adr = DEFAULT_CFG_MODEL.rdcu_model_adr;
+ cfg->rdcu_new_model_adr = DEFAULT_CFG_MODEL.rdcu_new_model_adr;
+ cfg->rdcu_buffer_adr = DEFAULT_CFG_MODEL.rdcu_buffer_adr;
+ } else {
+ cfg->rdcu_data_adr = DEFAULT_CFG_DIFF.rdcu_data_adr;
+ cfg->rdcu_model_adr = DEFAULT_CFG_DIFF.rdcu_model_adr;
+ cfg->rdcu_new_model_adr = DEFAULT_CFG_DIFF.rdcu_new_model_adr;
+ cfg->rdcu_buffer_adr = DEFAULT_CFG_DIFF.rdcu_buffer_adr;
+ }
+}
+
+
+/**
+ * @brief guess a good compression configuration
+ * @details use the referenced in the cfg struct (samples, input_buf, model_buf
+ * (optional)) and the cmp_mode to find a good set of compression parameters
+ * @note compression parameters in the cfg struct (golomb_par, spill, model_value,
+ * ap1_.., ap2_.., buffer_length, ...) are overwritten by this function
+ *
+ * @param cfg compression configuration structure
+ * @param level guess_level 1 -> fast; 2 -> default; 3 -> slow(brute force)
+ *
+ * @returns the size in bits of the compressed data of the guessed
+ * configuration; 0 on error
+ */
+
+uint32_t cmp_guess(struct cmp_cfg *cfg, int level)
+{
+ size_t size;
+ struct cmp_cfg work_cfg;
+ int err;
+ uint32_t cmp_size = 0;
+
+ if (!cfg)
+ return 0;
+
+ if (!cfg->input_buf)
+ return 0;
+
+ if (!rdcu_supported_mode_is_used(cfg->cmp_mode)) {
+ printf("This compression mode is not implied yet.\n");
+ return 0;
+ }
+ /* make a working copy of the input data (and model) because the
+ * following function works inplace */
+ work_cfg = *cfg;
+ work_cfg.input_buf = NULL;
+ work_cfg.model_buf = NULL;
+ work_cfg.icu_new_model_buf = NULL;
+ work_cfg.icu_output_buf = NULL;
+ work_cfg.buffer_length = 0;
+
+ size = cfg->samples * size_of_a_sample(work_cfg.cmp_mode);
+ if (size == 0)
+ goto error;
+ work_cfg.input_buf = malloc(size);
+ if (!work_cfg.input_buf) {
+ printf("malloc() failed!\n");
+ goto error;
+ }
+ memcpy(work_cfg.input_buf, cfg->input_buf, size);
+
+ if (cfg->model_buf && model_mode_is_used(cfg->cmp_mode)) {
+ work_cfg.model_buf = malloc(size);
+ if (!work_cfg.model_buf) {
+ printf("malloc() failed!\n");
+ goto error;
+ }
+ memcpy(work_cfg.model_buf, cfg->model_buf, size);
+
+ work_cfg.icu_new_model_buf = malloc(size);
+ if (!work_cfg.icu_new_model_buf) {
+ printf("malloc() failed!\n");
+ goto error;
+ }
+ }
+
+ err = cmp_pre_process(&work_cfg);
+ if (err)
+ goto error;
+
+ err = cmp_map_to_pos(&work_cfg);
+ if (err)
+ goto error;
+
+ /* find the best parameters */
+ switch (level) {
+ case 3:
+ cmp_size = brute_force(&work_cfg);
+ break;
+ case 1:
+ printf("guess level 1 not implied yet use guess level 2\n");
+ /* fall through */
+ case 2:
+ cmp_size = pre_cal_method(&work_cfg);
+ break;
+ default:
+ fprintf(stderr, "cmp_tool: guess level not supported!\n");
+ goto error;
+ break;
+ }
+ if (!cmp_size)
+ goto error;
+
+ free(work_cfg.input_buf);
+ free(work_cfg.model_buf);
+ free(work_cfg.icu_new_model_buf);
+
+ cfg->golomb_par = work_cfg.golomb_par;
+ cfg->spill = work_cfg.spill;
+
+ cfg->model_value = cmp_guess_model_value(num_model_updates);
+
+ if (rdcu_supported_mode_is_used(cfg->cmp_mode))
+ add_rdcu_pars_internal(cfg);
+
+ cfg->buffer_length = ((cmp_size + 32)&~0x1FU)/(size_of_a_sample(work_cfg.cmp_mode)*8);
+
+ return cmp_size;
+
+error:
+ free(work_cfg.input_buf);
+ free(work_cfg.model_buf);
+ free(work_cfg.icu_new_model_buf);
+ return 0;
+}
+
diff --git a/lib/cmp_icu.c b/lib/cmp_icu.c
index 1a8d1574c08e6551f5fa4baf429eed9261c60a98..21aabe7d9e444ca494a65f4720a1b2c59f4159f2 100644
--- a/lib/cmp_icu.c
+++ b/lib/cmp_icu.c
@@ -30,7 +30,6 @@
#include "../include/cmp_debug.h"
-
/**
* @brief check if the compressor configuration is valid for a SW compression,
* see the user manual for more information (PLATO-UVIE-PL-UM-0001).
@@ -62,9 +61,8 @@ int icu_cmp_cfg_valid(const struct cmp_cfg *cfg, struct cmp_info *info)
cfg_invalid++;
}
- if (cfg->samples == 0) {
+ if (cfg->samples == 0)
debug_print("Warning: The samples parameter is 0. No data are compressed. This behavior may not be intended.\n");
- }
/* icu_output_buf can be NULL if rdcu compression is used */
if (cfg->icu_output_buf == NULL) {
@@ -226,7 +224,7 @@ static int set_info(struct cmp_cfg *cfg, struct cmp_info *info)
if (cfg->model_value > UINT8_MAX)
return -1;
- if(info) {
+ if (info) {
info->cmp_err = 0;
info->cmp_mode_used = (uint8_t)cfg->cmp_mode;
info->model_value_used = (uint8_t)cfg->model_value;
@@ -310,7 +308,7 @@ static int diff_32(uint32_t *data_buf, unsigned int samples, unsigned int round)
* @note change the data_buf in-place
* @note output is I_0 = I_0, I_i = I_i - I_i-1, where i is the array index
*
- * @param data_buf pointer to a S_FX data buffer
+ * @param data pointer to a S_FX data buffer
* @param samples amount of data samples in the data buffer
* @param round number of bits to round; if zero no rounding takes place
*
@@ -344,7 +342,7 @@ static int diff_S_FX(struct S_FX *data, unsigned int samples, unsigned int
* @note change the data_buf in-place
* @note output is I_0 = I_0, I_i = I_i - I_i-1, where i is the array index
*
- * @param data_buf pointer to a S_FX_EFX data buffer
+ * @param data pointer to a S_FX_EFX data buffer
* @param samples amount of data samples in the data buffer
* @param round number of bits to round; if zero no rounding takes place
*
@@ -378,7 +376,7 @@ static int diff_S_FX_EFX(struct S_FX_EFX *data, unsigned int samples, unsigned
* @note change the data_buf in-place
* @note output is I_0 = I_0, I_i = I_i - I_i-1, where i is the array index
*
- * @param data_buf pointer to a S_FX_NCOB data buffer
+ * @param data pointer to a S_FX_NCOB data buffer
* @param samples amount of data samples in the data buffer
* @param round number of bits to round; if zero no rounding takes place
*
@@ -412,7 +410,7 @@ static int diff_S_FX_NCOB(struct S_FX_NCOB *data, unsigned int samples, unsigned
* @note change the data_buf in-place
* @note output is I_0 = I_0, I_i = I_i - I_i-1, where i is the array index
*
- * @param data_buf pointer to a S_FX_EFX_NCOB_ECOB data buffer
+ * @param data pointer to a S_FX_EFX_NCOB_ECOB data buffer
* @param samples amount of data samples in the data buffer
* @param round number of bits to round; if zero no rounding takes place
*
@@ -540,7 +538,6 @@ static int model_32(uint32_t *data_buf, uint32_t *model_buf, unsigned int sample
* @note update the model_buf in-place if up_model_buf = NULL
*
* @param data_buf pointer to the S_FX data buffer to process
- * @param model_buf pointer to the model buffer of the data to process
* @param model_buf pointer to the updated model buffer (if NULL model_buf
* will be overwrite with the updated model)
* @param samples amount of data samples in the data_buf and model_buf buffer
@@ -607,7 +604,7 @@ int model_S_FX(struct S_FX *data_buf, struct S_FX *model_buf,
* @returns 0 on success, error otherwise
*/
-int pre_process(struct cmp_cfg *cfg)
+int cmp_pre_process(struct cmp_cfg *cfg)
{
if (!cfg)
return -1;
@@ -934,13 +931,12 @@ static int map_to_pos_S_FX_EFX_NCOB_ECOB(struct S_FX_EFX_NCOB_ECOB *data_buf,
*
* @note change the data_buf in-place
*
- * @param data_buf pointer to the data to process
- * @param buf_len length of the data to process
+ * @param cfg configuration contains all parameters required for compression
*
* @returns 0 on success, error otherwise
*/
-static int map_to_pos(struct cmp_cfg *cfg)
+int cmp_map_to_pos(struct cmp_cfg *cfg)
{
int zero_mode_used;
@@ -1114,7 +1110,7 @@ static encoder_ptr select_encoder(unsigned int golomb_par)
* @param nBits number of bits to put in the bitstream
* @param destAddr this is the pointer to the beginning of the bitstream
* @param dest_len length of the bitstream buffer (starting at destAddr)
- * @returns number of bits written, 0 if the number was too big, -1 if the
+ * @returns TODO number of bits written, 0 if the number was too big, -2 if the
* destAddr buffer is to small to store the bitstream
* @note works in SRAM2
*/
@@ -1127,6 +1123,9 @@ static unsigned int put_n_bits32(unsigned int value, unsigned int bitOffset,
unsigned int bitsLeft, shiftRight, shiftLeft, localEndPos;
unsigned int mask;
+ if (!destAddr)
+ return nBits;
+
/* check if destination buffer is large enough */
/* TODO: adapt that to the other science products */
if ((bitOffset + nBits) > (((dest_len+1)/2)*2 * 16)) {
@@ -1336,9 +1335,8 @@ static int encode_outlier_zero(uint32_t value_to_encode, int max_bits,
/* put the data unencoded in the bitstream */
err = put_n_bits32(value_to_encode, enc->cmp_size, max_bits,
cfg->icu_output_buf, cfg->buffer_length);
- if (err <= 0) {
+ if (err <= 0)
return err;
- }
enc->cmp_size += max_bits;
@@ -1449,23 +1447,18 @@ int encode_value(uint32_t value_to_encode, int bit_len, struct cmp_cfg *cfg,
}
-static int encode_16(uint16_t *data_to_encode, struct cmp_cfg *cfg,
- struct encoder_struct *enc)
+static int encode_16(struct cmp_cfg *cfg, struct encoder_struct *enc)
{
size_t i;
+ uint16_t *data_to_encode;
if (!cfg)
return -1;
- if (!cfg->samples)
- return 0;
-
- if (!data_to_encode)
- return -1;
-
if (!enc)
return -1;
+ data_to_encode = cfg->input_buf;
for (i = 0; i < cfg->samples; i++) {
int err = encode_value(data_to_encode[i], 16, cfg, enc);
@@ -1614,7 +1607,7 @@ static int encode_S_FX_EFX_NCOB_ECOB(struct cmp_cfg *cfg, struct encoder_struct
}
/* pad the bitstream with zeros */
-int pad_bitstream(struct cmp_cfg *cfg, uint32_t cmp_size, struct cmp_info *info)
+int pad_bitstream(struct cmp_cfg *cfg, uint32_t cmp_size)
{
int n_bits = 0;
@@ -1628,21 +1621,15 @@ int pad_bitstream(struct cmp_cfg *cfg, uint32_t cmp_size, struct cmp_info *info)
n_bits = put_n_bits32(0, cmp_size, n_pad_bits,
cfg->icu_output_buf,
cfg->buffer_length);
- if (n_bits <= 0) {
- /* the icu_output_buf is to small to store the whole bitstream */
- if (info) {
- info->cmp_err |= 1UL << SMALL_BUFFER_ERR_BIT; /* set small buffer error */
- info->cmp_size = 0;
- }
+ if (n_bits <= 0)
return -2;
- }
}
}
return n_bits;
}
-static int encode_data(struct cmp_cfg *cfg, struct cmp_info *info)
+uint32_t cmp_encode_data(struct cmp_cfg *cfg)
{
struct encoder_struct enc;
int err, n_bits;
@@ -1659,7 +1646,7 @@ static int encode_data(struct cmp_cfg *cfg, struct cmp_info *info)
case MODE_MODEL_MULTI:
case MODE_DIFF_ZERO:
case MODE_DIFF_MULTI:
- err = encode_16((uint16_t *)cfg->input_buf, cfg, &enc);
+ err = encode_16(cfg, &enc);
break;
case MODE_RAW_S_FX:
err = encode_raw_S_FX(cfg, &enc);
@@ -1704,28 +1691,24 @@ static int encode_data(struct cmp_cfg *cfg, struct cmp_info *info)
break;
}
- if (err == -2) {
- /* the icu_output_buf is to small to store the whole bitstream */
- info->cmp_err |= 1UL << SMALL_BUFFER_ERR_BIT; /* set small buffer error */
- return err;
- }
- if (info)
- info->cmp_size = enc.cmp_size;
-
- n_bits = pad_bitstream(cfg, enc.cmp_size, info);
+ n_bits = pad_bitstream(cfg, enc.cmp_size);
if (n_bits < 0)
return n_bits;
#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
- {
- size_t i;
- uint32_t *p = (uint32_t *)cfg->icu_output_buf;
+ {
+ size_t i;
+ uint32_t *p = (uint32_t *)cfg->icu_output_buf;
+ if (p)
for (i = 0; i < (enc.cmp_size + n_bits)/32; i++)
cpu_to_be32s(&p[i]);
- }
+ }
#endif /*__BYTE_ORDER__ */
- return 0;
+ if (err)
+ return err;
+ else
+ return enc.cmp_size;
}
@@ -1756,6 +1739,7 @@ static int encode_data(struct cmp_cfg *cfg, struct cmp_info *info)
int icu_compress_data(struct cmp_cfg *cfg, struct cmp_info *info)
{
int err;
+ int cmp_size = 0;
err = set_info(cfg, info);
if (err)
@@ -1765,17 +1749,22 @@ int icu_compress_data(struct cmp_cfg *cfg, struct cmp_info *info)
if (err)
return err;
- err = pre_process(cfg);
+ err = cmp_pre_process(cfg);
if (err)
return err;
- err = map_to_pos(cfg);
+ err = cmp_map_to_pos(cfg);
if (err)
return err;
- err = encode_data(cfg, info);
- if (err)
- return err;
+ cmp_size = cmp_encode_data(cfg);
+ if (cmp_size == -2 && info)
+ /* the icu_output_buf is to small to store the whole bitstream */
+ info->cmp_err |= 1UL << SMALL_BUFFER_ERR_BIT; /* set small buffer error */
+ if (cmp_size < 0)
+ return cmp_size;
+ if (info)
+ info->cmp_size = cmp_size;
return 0;
}
diff --git a/lib/cmp_support.c b/lib/cmp_support.c
index 39aa61f69ce53691b54f9e7afaa13ccc23d0720d..c0783197adbfd4614e9c8cef6c2a898aa0abe1ec 100644
--- a/lib/cmp_support.c
+++ b/lib/cmp_support.c
@@ -17,7 +17,6 @@
* @see Data Compression User Manual PLATO-UVIE-PL-UM-0001
*/
-#include <stdio.h>
#include "../include/cmp_support.h"
#include "../include/cmp_data_types.h"
@@ -246,6 +245,26 @@ int rdcu_supported_mode_is_used(unsigned int cmp_mode)
}
+/**
+ * @brief check if the mode is available
+ *
+ * @param cmp_mode compression mode
+ *
+ * @returns 1 when mode is available, otherwise 0
+ */
+
+int cmp_mode_available(unsigned int cmp_mode)
+{
+ if (diff_mode_is_used(cmp_mode) ||
+ model_mode_is_used(cmp_mode) ||
+ raw_mode_is_used(cmp_mode))
+ return 1;
+ else
+ return 0;
+
+}
+
+
/**
* @brief check if zero escape symbol mechanism mode is used
*
@@ -391,28 +410,71 @@ unsigned int cal_up_model(unsigned int data, unsigned int model, unsigned int
uint32_t get_max_spill(unsigned int golomb_par, unsigned int cmp_mode)
{
- unsigned int cutoff;
- unsigned int max_n_sym_offset;
- unsigned int max_cw_bits;
+ const uint32_t LUT_MAX_RDCU[MAX_RDCU_GOLOMB_PAR+1] = { 0, 8, 22, 35, 48,
+ 60, 72, 84, 96, 107, 118, 129, 140, 151, 162, 173, 184, 194,
+ 204, 214, 224, 234, 244, 254, 264, 274, 284, 294, 304, 314, 324,
+ 334, 344, 353, 362, 371, 380, 389, 398, 407, 416, 425, 434, 443,
+ 452, 461, 470, 479, 488, 497, 506, 515, 524, 533, 542, 551, 560,
+ 569, 578, 587, 596, 605, 614, 623 };
if (golomb_par == 0)
return 0;
if (rdcu_supported_mode_is_used(cmp_mode)) {
- max_cw_bits = 16; /* the RDCU compressor can only generate code
- * words with a length of maximal 16 bits.
- */
+ if (golomb_par > MAX_RDCU_GOLOMB_PAR)
+ return 0;
+
+ return LUT_MAX_RDCU[golomb_par];
} else {
- max_cw_bits = 32; /* the ICU compressor can generate code
- * words with a length of maximal 32 bits.
- */
+ if (golomb_par > MAX_ICU_GOLOMB_PAR) {
+ return 0;
+ } else {
+ /* the ICU compressor can generate code words with a length of
+ * maximal 32 bits. */
+ unsigned int max_cw_bits = 32;
+ unsigned int cutoff = (1UL << (ilog_2(golomb_par)+1)) - golomb_par;
+ unsigned int max_n_sym_offset = max_cw_bits/2 - 1;
+ return (max_cw_bits-1-ilog_2(golomb_par))*golomb_par + cutoff -
+ max_n_sym_offset - 1;
+ }
}
+}
- cutoff = (1UL << (ilog_2(golomb_par)+1)) - golomb_par;
- max_n_sym_offset = max_cw_bits/2 - 1;
- return (max_cw_bits-1-ilog_2(golomb_par))*golomb_par + cutoff -
- max_n_sym_offset - 1;
+/**
+ * @brief get a good spill threshold parameter for the selected Golomb parameter
+ * and compression mode
+ *
+ * @param golomb_par Golomb parameter
+ * @param cmp_mode compression mode
+ *
+ * @returns a good spill parameter (optimal for zero escape mechanism)
+ * @warning icu compression not support yet!
+ */
+
+uint32_t cmp_get_good_spill(unsigned int golomb_par, unsigned int cmp_mode)
+{
+ const uint32_t LUT_RDCU_MULIT[MAX_RDCU_GOLOMB_PAR+1] = {0, 8, 16, 23,
+ 30, 36, 44, 51, 58, 64, 71, 77, 84, 90, 97, 108, 115, 121, 128,
+ 135, 141, 148, 155, 161, 168, 175, 181, 188, 194, 201, 207, 214,
+ 229, 236, 242, 250, 256, 263, 269, 276, 283, 290, 296, 303, 310,
+ 317, 324, 330, 336, 344, 351, 358, 363, 370, 377, 383, 391, 397,
+ 405, 411, 418, 424, 431, 452 };
+
+ if (zero_escape_mech_is_used(cmp_mode))
+ return get_max_spill(golomb_par, cmp_mode);
+
+ if (cmp_mode == MODE_MODEL_MULTI) {
+ if (golomb_par > MAX_RDCU_GOLOMB_PAR)
+ return 0;
+ else
+ return LUT_RDCU_MULIT[golomb_par];
+ }
+
+ if (cmp_mode == MODE_DIFF_MULTI)
+ return CMP_GOOD_SPILL_DIFF_MULTI;
+
+ return 0;
}
@@ -501,7 +563,7 @@ size_t size_of_a_sample(unsigned int cmp_mode)
*
* @param cmp_size compressed data size, measured in bits
*
- * @returns the size in bytes to sore the hole bitstream
+ * @returns the size in bytes to store the hole bitstream
*/
unsigned int size_of_bitstream(unsigned int cmp_size)
@@ -513,9 +575,10 @@ unsigned int size_of_bitstream(unsigned int cmp_size)
/**
* @brief calculate the need bytes for the model
*
- * @param cmp_size compressed data size, measured in bits
+ * @param samples amount of model samples
+ * @param cmp_mode compression mode
*
- * @returns the size in bytes to sore the hole bitstream
+ * @returns the size in bytes to store the hole bitstream
*/
unsigned int size_of_model(unsigned int samples, unsigned int cmp_mode)
@@ -548,18 +611,16 @@ void print_cmp_cfg(const struct cmp_cfg *cfg)
printf("input_buf: %p\n", (void *)cfg->input_buf);
if (cfg->input_buf != NULL) {
printf("input data:");
- for (i = 0; i < cfg->samples; i++) {
+ for (i = 0; i < cfg->samples; i++)
printf(" %04X", ((uint16_t *)cfg->input_buf)[i]);
- }
printf("\n");
}
printf("rdcu_data_adr: 0x%06X\n", cfg->rdcu_data_adr);
printf("model_buf: %p\n", (void *)cfg->model_buf);
if (cfg->model_buf != NULL) {
printf("model data:");
- for (i = 0; i < cfg->samples; i++) {
+ for (i = 0; i < cfg->samples; i++)
printf(" %04X", ((uint16_t *)cfg->model_buf)[i]);
- }
printf("\n");
}
printf("rdcu_model_adr: 0x%06X\n", cfg->rdcu_model_adr);
diff --git a/lib/tool_lib.c b/lib/cmp_tool_lib.c
similarity index 63%
rename from lib/tool_lib.c
rename to lib/cmp_tool_lib.c
index 5318366153f4c571b75fa1900b972c637aad67b8..41f7763f80ea0cff5e4dcb23325deac548820ccd 100644
--- a/lib/tool_lib.c
+++ b/lib/cmp_tool_lib.c
@@ -1,5 +1,5 @@
/**
- * @file tool_lib.c
+ * @file cmp_tool_lib.c
* @author Johannes Seelig (johannes.seelig@univie.ac.at)
* @author Dominik Loidolt (dominik.loidolt@univie.ac.at),
* @date 2020
@@ -22,7 +22,7 @@
#include <ctype.h>
#include <sys/stat.h>
-#include "../include/tool_lib.h"
+#include "../include/cmp_tool_lib.h"
#include "../include/cmp_support.h"
#include "../include/rdcu_cmd.h"
#include "../include/byteorder.h"
@@ -31,30 +31,35 @@
/**
* @brief print help information
*
- * @param argv argument vector
+ * @param program_name name of the program
*/
-void Print_Help(const char *argv)
+void print_help(const char *program_name)
{
- printf("usage: %s [options] [<argument>]\n", &argv[0]);
+ printf("usage: %s [options] [<argument>]\n", program_name);
printf("General Options:\n");
printf(" -h, --help Print this help text and exit\n");
printf(" -V, --version Print program version and exit\n");
printf(" -v, --verbose Print various debugging information\n");
printf(" -n, --model_cfg Print a default model configuration and exit\n");
printf(" --diff_cfg Print a default 1d-differencing configuration and exit\n");
- printf(" -a, --rdcu_par Print additional RDCU control parameters\n");
+ printf(" -a, --rdcu_par Add additional RDCU control parameters\n");
printf(" -o <prefix> Use the <prefix> for output files\n");
printf("Compression Options:\n");
printf(" -c <file> File containing the compressing configuration\n");
printf(" -d <file> File containing the data to be compressed\n");
printf(" -m <file> File containing the model of the data to be compressed\n");
- printf(" --rdcu_pkt Generate RMAP packets for a RDCU compression\n");
+ printf(" --rdcu_pkt Generate RMAP packets for an RDCU compression\n");
+ printf(" --last_info <.info file> Generate RMAP packets for an RDCU compression with parallel read of the last results\n");
printf("Decompression Options:\n");
printf(" -i <file> File containing the decompression information\n");
printf(" -d <file> File containing the compressed data\n");
printf(" -m <file> File containing the model of the compressed data\n");
- printf("\n");
+ printf("Guessing Options:\n");
+ printf(" --guess <mode> Search for a good configuration for compression <mode>\n");
+ printf(" -d <file> File containing the data to be compressed\n");
+ printf(" -m <file> File containing the model of the data to be compressed\n");
+ printf(" --guess_level <level> Set guess level to <level> (optional)\n");
}
@@ -72,14 +77,13 @@ void Print_Help(const char *argv)
static FILE *open_file(const char *dirname, const char *filename)
{
- FILE *fp;
char *pathname;
int n;
if (!dirname)
return NULL;
- if(!filename)
+ if (!filename)
return NULL;
errno = 0;
@@ -104,9 +108,7 @@ static FILE *open_file(const char *dirname, const char *filename)
abort();
}
- fp = fopen(pathname, "w");
- free(pathname);
- return fp;
+ return fopen(pathname, "w");
}
@@ -144,7 +146,7 @@ int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char
}
for (i = 0; i < buf_len; i++) {
- fprintf(fp, "%02X %02X",buf[i] >> 8 ,buf[i] & 0xFF);
+ fprintf(fp, "%02X %02X", buf[i] >> 8, buf[i] & 0xFF);
if ((i + 1) % 16 == 0)
fprintf(fp, "\n");
else
@@ -157,7 +159,7 @@ int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char
if (verbose) {
printf("\n\n");
for (i = 0; i < buf_len; i++) {
- printf("%02X %02X",buf[i] >> 8 ,buf[i] & 0xFF);
+ printf("%02X %02X", buf[i] >> 8, buf[i] & 0xFF);
if ((i + 1) % 16 == 0)
printf("\n");
else
@@ -185,12 +187,12 @@ int write_to_file16(const uint16_t *buf, uint32_t buf_len, const char
* @returns 0 on success, error otherwise
*/
-int write_cmp_data_file(const void *buf, uint32_t buf_size, const char
- *output_prefix, const char *name_extension, int verbose)
+int write_cmp_data_file(const void *buf, uint32_t buf_size, const char *output_prefix,
+ const char *name_extension, int verbose)
{
unsigned int i;
FILE *fp;
- uint8_t *p = (uint8_t *)buf;
+ const uint8_t *p = (const uint8_t *)buf;
if (!buf)
abort();
@@ -248,7 +250,7 @@ static void remove_spaces(char *s)
do {
while (*d == ' ' || *d == '\t')
- ++d;
+ d++;
} while ((*s++ = *d++) != '\0');
}
@@ -326,7 +328,7 @@ static int sram_addr_to_int(const char *addr)
* @see https://eternallyconfuzzled.com/atoi-is-evil-c-learn-why-atoi-is-awful
*/
-static int atoui32(const char *dep_str , const char *val_str, uint32_t *red_val)
+static int atoui32(const char *dep_str, const char *val_str, uint32_t *red_val)
{
long temp;
char *end = NULL;
@@ -340,14 +342,14 @@ static int atoui32(const char *dep_str , const char *val_str, uint32_t *red_val)
if (!red_val)
return -1;
+ errno = 0;
temp = strtol(val_str, &end, 10);
- if (end != val_str && errno != ERANGE && temp >= 0 && temp <=
- UINT32_MAX) {
+ if (end != val_str && errno != ERANGE && temp >= 0 &&
+ temp <= UINT32_MAX) {
*red_val = (uint32_t)temp;
return 0;
} else {
- fprintf(stderr, "%s: Error read in %s.\n", PROGRAM_NAME,
- dep_str);
+ fprintf(stderr, "%s: Error read in %s.\n", PROGRAM_NAME, dep_str);
*red_val = 0;
return -1;
}
@@ -355,14 +357,64 @@ static int atoui32(const char *dep_str , const char *val_str, uint32_t *red_val)
/**
-* @brief parse a file containing a compressing configuration
-*
-* @param fp FILE pointer
-* @param cfg compression configuration structure holding the read in
-* configuration
-*
-* @returns 0 on success, error otherwise
-*/
+ * @brief parse a compression mode vale string to a integer
+ * @note string can be either a number or the name of the compression mode
+ *
+ * @param cmp_mode_str string containing the compression mode value to parse
+ * @param cmp_mode address where the parsed result is written
+ *
+ * @returns 0 on success, error otherwise
+ */
+
+uint32_t cmp_mode_parse(const char *cmp_mode_str, uint32_t *cmp_mode)
+{
+ size_t j;
+ static const struct {
+ uint32_t cmp_mode;
+ const char *str;
+ } conversion[] = {
+ {MODE_RAW, "MODE_RAW"},
+ {MODE_MODEL_ZERO, "MODE_MODEL_ZERO"},
+ {MODE_DIFF_ZERO, "MODE_DIFF_ZERO"},
+ {MODE_MODEL_MULTI, "MODE_MODEL_MULTI"},
+ {MODE_DIFF_MULTI, "MODE_DIFF_MULTI"},
+ };
+
+ if (!cmp_mode_str)
+ return -1;
+ if (!cmp_mode)
+ return -1;
+
+ if (isalpha(cmp_mode_str[0])) { /* check if mode is given as text */
+ for (j = 0; j < sizeof(conversion) / sizeof(conversion[0]); ++j) {
+ if (!strcmp(cmp_mode_str, conversion[j].str)) {
+ *cmp_mode = conversion[j].cmp_mode;
+ return 0;
+ }
+ }
+ return -1;
+ } else {
+ if (atoui32(cmp_mode_str, cmp_mode_str, cmp_mode))
+ return -1;
+ }
+
+ if (!cmp_mode_available(*cmp_mode))
+ return -1;
+
+ return 0;
+}
+
+
+/**
+ * @brief parse a file containing a compressing configuration
+ * @note internal use only!
+ *
+ * @param fp FILE pointer
+ * @param cfg compression configuration structure holding the read in
+ * configuration
+ *
+ * @returns 0 on success, error otherwise
+ */
static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
{
@@ -379,7 +431,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
if (line[0] == '#' || line[0] == '\n')
continue; /* skip #'ed or empty lines */
- if (!strchr(line, '\n')){ /* detect a to long line */
+ if (!strchr(line, '\n')) { /* detect a to long line */
fprintf(stderr, "%s: Error read in line to long. Maximal line length is %d characters.\n",
PROGRAM_NAME, MAX_CONFIG_LINE-1);
return -1;
@@ -538,6 +590,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en)
{
+ int err;
FILE *fp;
if (!file_name)
@@ -547,8 +600,8 @@ int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en)
abort();
if (strstr(file_name, ".info")) {
- fprintf(stderr, "%s: %s: .info file extension found on configuration file. You may have selected the wrong file.\n",
- PROGRAM_NAME, file_name);
+ fprintf(stderr, "%s: %s: .info file extension found on configuration file. You may have selected the wrong file.\n",
+ PROGRAM_NAME, file_name);
}
fp = fopen(file_name, "r");
@@ -559,10 +612,10 @@ int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en)
return -1;
}
- if (parse_cfg(fp, cfg))
- return -1;
-
+ err = parse_cfg(fp, cfg);
fclose(fp);
+ if (err)
+ return err;
if (verbose_en) {
printf("\n\n");
@@ -582,6 +635,7 @@ int read_cmp_cfg(const char *file_name, struct cmp_cfg *cfg, int verbose_en)
/**
* @brief parse a file containing a decompression information
+ * @note internal use only!
*
* @param fp FILE pointer
* @param info decompression information structure holding the read in
@@ -605,7 +659,7 @@ static int parse_info(FILE *fp, struct cmp_info *info)
if (line[0] == '#' || line[0] == '\n')
continue; /* skip #'ed or empty lines */
- if (!strchr(line, '\n')){ /* detect a to long line */
+ if (!strchr(line, '\n')) { /* detect a to long line */
fprintf(stderr, "%s: Error read in line to long. Maximal line length is %d characters.\n",
PROGRAM_NAME, MAX_CONFIG_LINE-1);
return -1;
@@ -738,8 +792,7 @@ static int parse_info(FILE *fp, struct cmp_info *info)
int i = sram_addr_to_int(token2);
if (i < 0) {
- fprintf(stderr,"%s: Error read in "
- "rdcu_cmp_adr_used\n",
+ fprintf(stderr, "%s: Error read in rdcu_cmp_adr_used\n",
PROGRAM_NAME);
return -1;
}
@@ -786,10 +839,9 @@ int read_cmp_info(const char *file_name, struct cmp_info *info, int verbose_en)
if (!info)
abort();
- if (strstr(file_name, ".cfg")) {
- fprintf(stderr, "%s: %s: .cfg file extension found on decompression information file. You may have selected the wrong file.\n",
- PROGRAM_NAME, file_name);
- }
+ if (strstr(file_name, ".cfg"))
+ fprintf(stderr, "%s: %s: .cfg file extension found on decompression information file. You may have selected the wrong file.\n",
+ PROGRAM_NAME, file_name);
fp = fopen(file_name, "r");
if (fp == NULL) {
@@ -813,297 +865,383 @@ int read_cmp_info(const char *file_name, struct cmp_info *info, int verbose_en)
/**
- * @brief reads a hex encoded uint8_t data (or model) file, returns a buffer
- * containing the read in data
+ * @brief reads n_word words of a hex-encoded data (or model) file to
+ * a uint8_t buffer
*
- * @note data must be encode has 4 hex digits separated by a white space or new
- * line character e.g. "ABCD 1234 AB12\n"
+ * @note data must be encoded as 2 hex digits separated by a white space or new
+ * line character e.g. "AB CD 12\n34 AB 12\n"
*
- * @param file_name data file name
- * @param samples amount of uint16_t data samples to read in
+ * @param file_name data/model file name
+ * @param buf buffer to write the file content (can be NULL)
+ * @param n_word number of uint8_t data words to read in
* @param verbose_en print verbose output if not zero
*
- * @returns address to the buffer containing the read in data, NULL on error
+ * @returns read data words; if buf == NULL the size in bytes to store the file
+ * content; negative on error
*/
-uint8_t *read_file8(const char *file_name, uint32_t samples, int verbose_en)
+ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int verbose_en)
{
- uint32_t i;
+ /* This is a rather slow implementation. Performance can be improved by
+ * reading larger chunks */
+ ssize_t n;
FILE *fp;
- uint8_t *buffer;
- char tmp_str[4]; /* 6 = 2 hex digit's + 1 white space + 1 \0 */
+ char tmp_str[4]; /* 4 = 2 hex digit's + 1 white space + 1 \0 */
if (!file_name)
- abort();
-
- if (!samples)
- return NULL;
+ return -1;
fp = fopen(file_name, "r");
if (fp == NULL) {
fprintf(stderr, "%s: %s: %s\n", PROGRAM_NAME, file_name,
strerror(errno));
- return NULL;
+ return -1;
}
- buffer = (uint8_t *)malloc(samples * SAM2BYT);
- if (buffer == NULL) {
- fprintf(stderr, "%s: Error allocating memory.\n", PROGRAM_NAME);
- fclose(fp);
- return NULL;
- }
+ /* if buf is NULL we count the words in the file */
+ if (!buf)
+ n_word = ~0U;
- for (i = 0; i < samples; i++) {
+ for (n = 0; n < n_word; ) {
int j;
unsigned long read_val;
char *end;
+ /* read 3 characters */
if (!fgets(tmp_str, sizeof(tmp_str), fp)) {
- fprintf(stderr, "%s: %s: Error: The files does not contain enough data as given by the samples parameter.\n",
+ if (ferror(fp)) {
+ fprintf(stderr, "%s: %s: Error: File error indicator set.\n",
+ PROGRAM_NAME, file_name);
+ goto error;
+ }
+
+ if (!buf) /* finished counting the sample */
+ break;
+
+ fprintf(stderr, "%s: %s: Error: The files does not contain enough data as given by the n_word parameter.\n",
PROGRAM_NAME, file_name);
goto error;
}
- if (tmp_str[0] == '#' || tmp_str[0] == '\n') {
- i--; /* a comment or empty line does not count as sample */
+ /* skip empty lines */
+ if (tmp_str[0] == '\n')
+ continue;
+
+ /* skip comment lines */
+ if (tmp_str[0] == '#') {
+ int c;
+ do {
+ c = fgetc(fp);
+ } while (c != '\n' && c != EOF);
continue;
}
+ /* check the string formation */
for (j = 0; j < 2; j++) {
if (!isxdigit(tmp_str[j])) {
- fprintf(stderr, "%s: %s: Error: The data are not correct formatted. Expected format is like: 12 AB 23 CD .. ..\n", PROGRAM_NAME,
- file_name);
+ fprintf(stderr, "%s: %s: Error: The data are not correct formatted. Expected format is like: 12 AB 23 CD .. ..\n",
+ PROGRAM_NAME, file_name);
goto error;
}
}
-
- if (!(isspace(tmp_str[2]) || tmp_str[2] == '\n')) {
- fprintf(stderr, "%s: %s: Error: The data are not correct formatted. Expected format is: 12 AB 23 CD .. ..\n", PROGRAM_NAME, file_name);
- goto error;
- }
-
- read_val = strtoul(tmp_str, &end, 16);
-
- if (tmp_str == end || errno == ERANGE || read_val > UINT8_MAX) {
- fprintf(stderr, "%s: %s: Error: The data can not be converted to integer.\n",
+ if (!isspace(tmp_str[2])) {
+ fprintf(stderr, "%s: %s: Error: The data are not correct formatted. Expected format is like: 12 AB 23 CD .. ..\n",
PROGRAM_NAME, file_name);
goto error;
}
- buffer[i] = (uint16_t)read_val;
-
- if (verbose_en) {
- if (i == 0)
- printf("\n\n");
-
- printf("%02X", buffer[i]);
- if (i && !((i + 1) % 28))
- printf("\n");
- else
- printf(" ");
+ /* convert data to integer and write it into the buffer */
+ if (buf) {
+ read_val = strtoul(tmp_str, &end, 16);
+ if (tmp_str == end || read_val > UINT8_MAX) {
+ fprintf(stderr, "%s: %s: Error: The data can not be converted to integer.\n",
+ PROGRAM_NAME, file_name);
+ goto error;
+ }
+ buf[n] = (uint8_t)read_val;
+
+ /* print data read in */;
+ if (verbose_en) {
+ if (n == 0)
+ printf("\n\n");
+ printf("%02X", buf[n]);
+ if (n && !((n + 1) % 32))
+ printf("\n");
+ else
+ printf(" ");
+ }
}
- }
- if (verbose_en)
- printf("\n\n");
- fgets(tmp_str, sizeof(tmp_str), fp);
- if (!feof(fp) && tmp_str[0] == '\n') /* read last line break */
- fgets(tmp_str, sizeof(tmp_str), fp);
+ n++;
+ }
- if (!feof(fp)) {
- fprintf(stderr, "%s: %s: Warning: The file may contain more data than specified by the samples parameter.\n",
- PROGRAM_NAME, file_name);
+ if (buf) {
+ /* check if there is some additional stuff at the end of the file */
+ int c = getc(fp);
+ if (c != EOF)
+ if (c != '\n' && getc(fp) != EOF)
+ fprintf(stderr, "%s: %s: Warning: The file may contain more data than specified by the samples or cmp_size parameter.\n",
+ PROGRAM_NAME, file_name);
}
fclose(fp);
- return buffer;
+ if (verbose_en && buf)
+ printf("\n\n");
+
+ return n;
- error:
- free(buffer);
- fclose(fp);
- return NULL;
+error:
+ fclose(fp);
+ return -1;
}
/**
- * @brief reads a hex encoded uint16_t data (or model) file, returns a buffer
- * containing the read in data
+ * @brief reads the number of uint16_t samples of a hex-encoded data (or model)
+ * file into a buffer
*
- * @note data must be encode has 4 hex digits separated by a white space or new
- * line character e.g. "ABCD 1234 AB12\n"
+ * @note data must be encode has 2 hex digits separated by a white space or new
+ * line character e.g. "AB CD 12 34 AB 12\n"
*
- * @param file_name data file name
+ * @param file_name data/model file name
+ * @param buf buffer to write the file content (can be NULL)
* @param samples amount of uint16_t data samples to read in
* @param verbose_en print verbose output if not zero
*
- * @returns address to the buffer containing the read in data, NULL on error
+ * @returns the size in bytes to store the file content; negative on error
*/
-uint16_t *read_file16(const char *file_name, uint32_t samples, int verbose_en)
+ssize_t read_file16(const char *file_name, uint16_t *buf, uint32_t samples,
+ int verbose_en)
{
- size_t i;
- uint16_t *buf = (uint16_t *)read_file8(file_name, samples*2, verbose_en);
+ ssize_t size = read_file8(file_name, (uint8_t *)buf,
+ samples*sizeof(uint16_t), verbose_en);
- if (!buf)
- return NULL;
+ if (size < 0)
+ return size;
- for (i=0; i < samples; i++)
- be16_to_cpus(&buf[i]);
+ if (size & 0x1) {
+ fprintf(stderr, "%s: %s: Error: The data are not correct formatted. Expected multiple of 2 hex words.\n",
+ PROGRAM_NAME, file_name);
+ return -1;
+ }
- return buf;
+ if (buf) {
+ size_t i;
+ for (i = 0; i < samples; i++)
+ be16_to_cpus(&buf[i]);
+ }
+
+
+ return size;
}
/**
- * @brief reads a hex encoded uint32_t data file, returns a buffer containing
- * the read in data
+ * @brief reads the number of uint32_t samples of a hex-encoded data (or model)
+ * file into a buffer
*
* @note data must be encode has 2 hex digits separated by a white space or new
- * line character e.g. AB CD 12 34
+ * line character e.g. "AB CD 12 34 AB 12\n"
*
- * @param file_name data file name
- * @param buf_len amount of uint32_t data samples to read in
+ * @param file_name data/model file name
+ * @param buf buffer to write the file content (can be NULL)
+ * @param samples amount of uint32_t data samples to read in
* @param verbose_en print verbose output if not zero
*
- * @returns address to the buffer containing the read in data, NULL on error
+ * @returns the size in bytes to store the file content; negative on error
*/
-uint32_t *read_file32(const char *file_name, uint32_t buf_len, int verbose_en)
+ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t samples,
+ int verbose_en)
{
- size_t i;
+ ssize_t size = read_file8(file_name, (uint8_t *)buf,
+ samples*sizeof(uint32_t), verbose_en);
- uint32_t *buf = (uint32_t *)read_file8(file_name, buf_len*sizeof(uint32_t),
- verbose_en);
+ if (size < 0)
+ return -1;
- if (!buf)
- return NULL;
+ if (size & 0x3) {
+ fprintf(stderr, "%s: %s: Error: The data are not correct formatted. Expected multiple of 4 hex words.\n",
+ PROGRAM_NAME, file_name);
+ return -1;
+ }
- for (i=0; i < buf_len; i++)
- be32_to_cpus(&buf[i]);
+ if (buf) {
+ size_t i;
+ for (i = 0; i < samples; i++)
+ be32_to_cpus(&buf[i]);
+ }
- return buf;
+ return size;
}
/**
- * @brief prints config struct
+ * @brief write compression configuration to a stream
+ * @note internal use only!
*
+ * @param fp FILE pointer
* @param cfg configuration to print
* @param rdcu_cfg if set additional RDCU parameter are printed as well
*/
-void print_cfg(const struct cmp_cfg *cfg, int rdcu_cfg)
+static void write_cfg_internal(FILE *fp, const struct cmp_cfg *cfg, int rdcu_cfg)
{
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Default Configuration File\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Selected compression mode\n");
- printf("# 0: raw mode\n");
- printf("# 1: model mode with zero escape symbol mechanism\n");
- printf("# 2: 1d differencing mode without input model with zero escape symbol mechanism\n");
- printf("# 3: model mode with multi escape symbol mechanism\n");
- printf("# 4: 1d differencing mode without input model multi escape symbol mechanism\n");
- printf("\n");
- printf("cmp_mode = %u\n", cfg->cmp_mode);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Number of samples (16 bit value) to compress, length of the data and model buffer\n");
- printf("\n");
- printf("samples = %u\n", cfg->samples);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Length of the compressed data buffer in number of samples (16 bit values)\n");
- printf("\n");
- printf("buffer_length = %u\n", cfg->buffer_length);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Golomb parameter for dictionary selection\n");
- printf("\n");
- printf("golomb_par = %u\n", cfg->golomb_par);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Spillover threshold for encoding outliers\n");
- printf("\n");
- printf("spill = %u\n", cfg->spill);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Model weighting parameter\n");
- printf("\n");
- printf("model_value = %u\n", cfg->model_value);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Number of noise bits to be rounded\n");
- printf("\n");
- printf("round = %u\n", cfg->round);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Default Configuration File\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Selected compression mode\n");
+ fprintf(fp, "# 0: raw mode\n");
+ fprintf(fp, "# 1: model mode with zero escape symbol mechanism\n");
+ fprintf(fp, "# 2: 1d differencing mode without input model with zero escape symbol mechanism\n");
+ fprintf(fp, "# 3: model mode with multi escape symbol mechanism\n");
+ fprintf(fp, "# 4: 1d differencing mode without input model multi escape symbol mechanism\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "cmp_mode = %u\n", cfg->cmp_mode);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Number of samples (16 bit value) to compress, length of the data and model buffer\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "samples = %u\n", cfg->samples);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Length of the compressed data buffer in number of samples (16 bit values)\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "buffer_length = %u\n", cfg->buffer_length);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Golomb parameter for dictionary selection\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "golomb_par = %u\n", cfg->golomb_par);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Spillover threshold for encoding outliers\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "spill = %u\n", cfg->spill);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Model weighting parameter\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "model_value = %u\n", cfg->model_value);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Number of noise bits to be rounded\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "round = %u\n", cfg->round);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
if (rdcu_cfg) {
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Hardware Compressor Settings (not need for SW compression)\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("\n");
- printf("# Adaptive 1 Golomb parameter; HW only\n");
- printf("\n");
- printf("ap1_golomb_par = %u\n", cfg->ap1_golomb_par);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Adaptive 1 spillover threshold; HW only\n");
- printf("\n");
- printf("ap1_spill = %u\n", cfg->ap1_spill);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Adaptive 2 Golomb parameter; HW only\n");
- printf("\n");
- printf("ap2_golomb_par = %u\n", cfg->ap2_golomb_par);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# Adaptive 2 spillover threshold; HW only\n");
- printf("\n");
- printf("ap2_spill = %u\n", cfg->ap2_spill);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# RDCU data to compress start address, the first data address in the RDCU SRAM; HW only\n");
- printf("\n");
- printf("rdcu_data_adr = 0x%06X\n",
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Hardware Compressor Settings (not need for SW compression)\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "# Adaptive 1 Golomb parameter; HW only\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "ap1_golomb_par = %u\n", cfg->ap1_golomb_par);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Adaptive 1 spillover threshold; HW only\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "ap1_spill = %u\n", cfg->ap1_spill);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Adaptive 2 Golomb parameter; HW only\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "ap2_golomb_par = %u\n", cfg->ap2_golomb_par);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# Adaptive 2 spillover threshold; HW only\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "ap2_spill = %u\n", cfg->ap2_spill);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# RDCU data to compress start address, the first data address in the RDCU SRAM; HW only\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "rdcu_data_adr = 0x%06X\n",
cfg->rdcu_data_adr);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# RDCU model start address, the first model address in the RDCU SRAM\n");
- printf("\n");
- printf("rdcu_model_adr = 0x%06X\n",
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# RDCU model start address, the first model address in the RDCU SRAM\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "rdcu_model_adr = 0x%06X\n",
cfg->rdcu_model_adr);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# RDCU updated model start address, the address in the RDCU SRAM where the updated model is stored\n");
- printf("\n");
- printf("rdcu_new_model_adr = 0x%06X\n",
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# RDCU updated model start address, the address in the RDCU SRAM where the updated model is stored\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "rdcu_new_model_adr = 0x%06X\n",
cfg->rdcu_new_model_adr);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
- printf("# RDCU compressed data start address, the first output data address in the RDCU SRAM\n");
- printf("\n");
- printf("rdcu_buffer_adr = 0x%06X\n", cfg->rdcu_buffer_adr);
- printf("\n");
- printf("#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ fprintf(fp, "# RDCU compressed data start address, the first output data address in the RDCU SRAM\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "rdcu_buffer_adr = 0x%06X\n", cfg->rdcu_buffer_adr);
+ fprintf(fp, "\n");
+ fprintf(fp, "#-------------------------------------------------------------------------------\n");
+ }
+}
+
+/**
+ * @brief prints config struct
+ *
+ * @param cfg configuration to print
+ * @param rdcu_cfg if set additional RDCU parameter are printed as well
+ */
+
+void print_cfg(const struct cmp_cfg *cfg, int rdcu_cfg)
+{
+ write_cfg_internal(stdout, cfg, rdcu_cfg);
+}
+
+/**
+ * @brief write compression configuration to a file
+ *
+ * @param cfg configuration to print
+ * @param output_prefix prefix of the written file (.cfg is added to the prefix)
+ * @param rdcu_cfg if non-zero additional RDCU parameter are printed as well
+ * @param verbose print verbose output if not zero
+ *
+ * @returns 0 on success, error otherwise
+ */
+
+int write_cfg(const struct cmp_cfg *cfg, const char *output_prefix, int rdcu_cfg,
+ int verbose)
+{
+ FILE *fp = open_file(output_prefix, ".cfg");
+
+ if (fp == NULL) {
+ fprintf(stderr, "%s: %s%s: %s\n", PROGRAM_NAME, output_prefix,
+ ".cfg", strerror(errno));
+ return -1;
}
+
+ write_cfg_internal(fp, cfg, rdcu_cfg);
+
+ fclose(fp);
+
+ if (verbose)
+ print_cfg(cfg, rdcu_cfg);
+ return 0;
}
+
/**
* @brief write a decompression information structure to a file
*
* @param info compressor information contains information of an executed
* compression
* @param output_prefix prefix of the written file (.info is added to the prefix)
- * @param machine_cfg - if set write additional RDCU parameter in the file
+ * @param rdcu_cfg - if non-zero write additional RDCU parameter in the file
*
* @returns 0 on success, error otherwise
*/
int write_info(const struct cmp_info *info, const char *output_prefix,
- int machine_cfg)
+ int rdcu_cfg)
{
FILE *fp = open_file(output_prefix, ".info");
@@ -1157,7 +1295,7 @@ int write_info(const struct cmp_info *info, const char *output_prefix,
fprintf(fp, "\n");
fprintf(fp, "#-------------------------------------------------------------------------------\n");
- if (machine_cfg) {
+ if (rdcu_cfg) {
fprintf(fp, "#-------------------------------------------------------------------------------\n");
fprintf(fp, "# Hardware Compressor Settings (not need for SW compression)\n");
fprintf(fp, "#-------------------------------------------------------------------------------\n");
diff --git a/lib/decmp.c b/lib/decmp.c
index 11387726ca69e7025afc56b63ef4cfb6e84bbc4a..e735c3db5fd85a9b054eb90ad48a5304e76023d2 100644
--- a/lib/decmp.c
+++ b/lib/decmp.c
@@ -57,7 +57,7 @@ static int de_raw_pre_process(uint8_t cmp_mode_used)
/**
* @brief model decompression pre-processing
*
- * @note
+ * @note change the data_buf in-place
*
* @param data_buf pointer to the data to process
* @param model_buf pointer to the model of the data to process
diff --git a/lib/rdcu_pkt_to_file.c b/lib/rdcu_pkt_to_file.c
index 24c213ee74134662530d113f80224d1151cf51c8..ea6ff5599a6682b853ce3576a0d9b25152ab7407 100644
--- a/lib/rdcu_pkt_to_file.c
+++ b/lib/rdcu_pkt_to_file.c
@@ -228,6 +228,7 @@ static int read_rdcu_pkt_mode_cfg(uint8_t *icu_addr, uint8_t *rdcu_addr,
*p = '\0';
} else {
fprintf(stderr, "Error read in line to long.\n");
+ fclose(fp);
return -1;
}
@@ -241,6 +242,7 @@ static int read_rdcu_pkt_mode_cfg(uint8_t *icu_addr, uint8_t *rdcu_addr,
i > 0xFF) {
fprintf(stderr, "Error reading ICU_ADDR.\n");
errno = 0;
+ fclose(fp);
return -1;
}
*icu_addr = (uint8_t)i;
@@ -255,6 +257,7 @@ static int read_rdcu_pkt_mode_cfg(uint8_t *icu_addr, uint8_t *rdcu_addr,
|| i > 0xFF) {
fprintf(stderr, "Error reading RDCU_ADDR.\n");
errno = 0;
+ fclose(fp);
return -1;
}
*rdcu_addr = (uint8_t)i;
@@ -269,6 +272,7 @@ static int read_rdcu_pkt_mode_cfg(uint8_t *icu_addr, uint8_t *rdcu_addr,
i > INT_MAX) {
fprintf(stderr, "Error reading MTU.\n");
errno = 0;
+ fclose(fp);
return -1;
}
*mtu = (int)i;
diff --git a/lib/rdcu_rmap.c b/lib/rdcu_rmap.c
index e25241c19e1e6c41034776ae24fa79576d1d99df..131cea07b0f4dee9f670be3dbb86a98bafdeeee6 100644
--- a/lib/rdcu_rmap.c
+++ b/lib/rdcu_rmap.c
@@ -261,7 +261,7 @@ static int rdcu_process_rx(void)
cnt++;
- if (0)
+ if ((0))
rmap_parse_pkt(spw_pckt);
/* convert format */
@@ -353,7 +353,7 @@ int rdcu_submit_tx(const uint8_t *cmd, int cmd_size,
if (!rmap_tx)
return -1;
- if (0)
+ if ((0))
printf("Transmitting RMAP command\n");
if (rmap_tx(cmd, cmd_size, dpath_len, data, data_size)) {
@@ -530,8 +530,8 @@ int rdcu_sync_data(int (*fn)(uint16_t trans_id, uint8_t *cmd,
slot = trans_log_grab_slot(data);
if (slot < 0) {
- if (0)
- printf("Error: all slots busy!\n");
+ if ((0))
+ printf("Error: all slots busy!\n");
return 1;
}
@@ -792,10 +792,10 @@ void rdcu_rmap_reset_log(void)
*
* @param mtu the maximum data transfer size per unit
*
- * @param rmap_tx a function pointer to transmit an rmap command
- * @param rmap_rx function pointer to receive an rmap command
+ * @param tx a function pointer to transmit an rmap command
+ * @param rx function pointer to receive an rmap command
*
- * @note rmap_tx is expected to return 0 on success
+ * @note tx is expected to return 0 on success
* rmap_rx is expected to return the number of packet bytes
*
* @returns 0 on success, otherwise error
diff --git a/test/cmp_tool/Makefile b/test/cmp_tool/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..543ea135a4f15e10d25440b3a7e17c9d3e1b41ff
--- /dev/null
+++ b/test/cmp_tool/Makefile
@@ -0,0 +1,25 @@
+CC = gcc
+SOURCEDIR =
+INCLUDES = -I../../include
+PATH +=
+CFLAGS := -O0 -W -Wall -Wextra -std=gnu99 -Werror -pedantic -g3\
+ -fprofile-arcs -ftest-coverage
+CPPFLAGS := -D__MAIN__ $(INCLUDES) -I$(SOURCEDIR)
+LDFLAGS := -lcunit
+SOURCES := $(wildcard *.c) ../../lib/cmp_tool_lib.c ../../lib/cmp_guess.c\
+ ../../lib/cmp_support.c ../../lib/cmp_icu.c ../../lib/cmp_data_types.c
+OBJECTS := $(patsubst %.c, $(BUILDDIR)/%.o, $(subst $(SOURCEDIR)/,, $(SOURCES)))
+TARGET := test_cmp_tool
+
+
+all: $(SOURCES)
+ $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $^ -o $(TARGET)
+
+coverage: all
+ ./$(TARGET)
+ lcov --rc lcov_branch_coverage=1 --capture --directory ./ --output-file coverage.info
+ genhtml --rc lcov_branch_coverage=1 --branch-coverage coverage.info --output-directory out
+ #firefox out/index.html
+
+clean:
+ rm -r $(TARGET) *.gcno *.gcda coverage.info out/ tmp_stderr.log
diff --git a/test/cmp_tool/cmp_tool_integration_test.py b/test/cmp_tool/cmp_tool_integration_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..60e13cbb96b1a21734340f3157a0ee070976c4e0
--- /dev/null
+++ b/test/cmp_tool/cmp_tool_integration_test.py
@@ -0,0 +1,535 @@
+import subprocess
+import shlex
+import sys
+import os
+
+PATH_CMP_TOOL = "./cmp_tool"
+VERSION = "0.06"
+
+EXIT_FAILURE = 1
+EXIT_SUCCESS = 0
+
+
+def call_cmp_tool(args):
+ args = shlex.split(PATH_CMP_TOOL + " " + args)
+
+ try:
+ with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) as proc:
+ stdout, stderr = proc.communicate()
+ return (proc.returncode, stdout, stderr)
+ except:
+ print('Could not find the cmp_tool or has no execute rights!')
+ assert()
+
+
+def parse_key_value(str):
+ dir = {}
+ for line in str.splitlines():
+ if line == '':
+ continue
+ if line[0] == '#' or line[0] == '\n':
+ continue
+ name, var = line.partition(" = ")[::2]
+ dir[name.strip()] = var
+ return dir
+
+
+def del_file(filePath):
+ if os.path.exists(filePath):
+ os.remove(filePath)
+ else:
+ print("The file %s could not be deleted because it does not exist!" % (filePath))
+
+
+HELP_STRING = \
+"""usage: %s [options] [<argument>]
+General Options:
+ -h, --help Print this help text and exit
+ -V, --version Print program version and exit
+ -v, --verbose Print various debugging information
+ -n, --model_cfg Print a default model configuration and exit
+ --diff_cfg Print a default 1d-differencing configuration and exit
+ -a, --rdcu_par Add additional RDCU control parameters
+ -o <prefix> Use the <prefix> for output files
+Compression Options:
+ -c <file> File containing the compressing configuration
+ -d <file> File containing the data to be compressed
+ -m <file> File containing the model of the data to be compressed
+ --rdcu_pkt Generate RMAP packets for an RDCU compression
+ --last_info <.info file> Generate RMAP packets for an RDCU compression with parallel read of the last results
+Decompression Options:
+ -i <file> File containing the decompression information
+ -d <file> File containing the compressed data
+ -m <file> File containing the model of the compressed data
+Guessing Options:
+ --guess <mode> Search for a good configuration for compression <mode>
+ -d <file> File containing the data to be compressed
+ -m <file> File containing the model of the data to be compressed
+ --guess_level <level> Set guess level to <level> (optional)
+""" % (PATH_CMP_TOOL)
+
+
+CMP_START_STR = \
+"""#########################################################
+### PLATO Compression/Decompression Tool Version %s ###
+#########################################################
+
+""" % (VERSION)
+
+
+def test_no_option():
+ returncode, stdout, stderr = call_cmp_tool("")
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == HELP_STRING)
+ assert(stderr == "")
+
+
+def test_invalid_option():
+ args = ['-q', '--not_used']
+ for arg in args:
+ returncode, stdout, stderr = call_cmp_tool(arg)
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == HELP_STRING)
+ if arg == '-q':
+ if sys.platform == 'linux':
+ assert(stderr == "%s: invalid option -- 'q'\n" % (PATH_CMP_TOOL))
+ elif "win" in sys.platform:
+ assert(stderr == "%s: unknown option -- q\n" % (PATH_CMP_TOOL))
+ else:
+ assert(stderr == "cmp_tool: invalid option -- q\n")
+ else:
+ if sys.platform == 'linux':
+ assert(stderr == "%s: unrecognized option '--not_used'\n" % (PATH_CMP_TOOL))
+ elif "win" in sys.platform:
+ assert(stderr == "%s: unknown option -- not_used\n" % (PATH_CMP_TOOL))
+ else:
+ assert(stderr == "cmp_tool: unrecognized option `--not_used'\n")
+
+# def option requires an argument
+
+
+def test_help():
+ args = ['-h', '--help']
+ for arg in args:
+ returncode, stdout, stderr = call_cmp_tool(arg)
+ assert(returncode == EXIT_SUCCESS)
+ assert(stdout == HELP_STRING)
+ assert(stderr == "")
+
+
+def test_version():
+ args = ['-V', '--version']
+ for arg in args:
+ returncode, stdout, stderr = call_cmp_tool(arg)
+ assert(returncode == EXIT_SUCCESS)
+ assert(stdout == "cmp_tool version %s\n" % (VERSION))
+ assert(stderr == "")
+
+
+def test_print_diff_cfg():
+ # comments in the cfg file is not checked
+ args = ['--diff_cfg', '--diff_cfg -a', '--diff_cfg --rdcu_par']
+ for i, arg in enumerate(args):
+ returncode, stdout, stderr = call_cmp_tool(arg)
+ assert(returncode == EXIT_SUCCESS)
+ assert(stderr == "")
+
+ cfg = parse_key_value(stdout)
+ assert(cfg['cmp_mode'] == '2')
+ assert(cfg['samples'] == '0')
+ assert(cfg['buffer_length'] == '0')
+ assert(cfg['golomb_par'] == '7')
+ assert(cfg['spill'] == '60')
+ assert(cfg['model_value'] == '8')
+ assert(cfg['round'] == '0')
+ if i > 0:
+ assert(cfg['ap1_golomb_par'] == '6')
+ assert(cfg['ap1_spill'] == '48')
+ assert(cfg['ap2_golomb_par'] == '8')
+ assert(cfg['ap2_spill'] == '72')
+ assert(cfg['rdcu_data_adr'] == '0x000000')
+ assert(cfg['rdcu_model_adr'] == '0x000000')
+ assert(cfg['rdcu_new_model_adr'] == '0x000000')
+ assert(cfg['rdcu_buffer_adr'] == '0x600000')
+
+
+def test_print_model_cfg():
+ # comments in the cfg file is not checked
+ args = ['-n', '--model_cfg', '-na', '--model_cfg --rdcu_par']
+ for i, arg in enumerate(args):
+ returncode, stdout, stderr = call_cmp_tool(arg)
+ assert(returncode == EXIT_SUCCESS)
+ assert(stderr == "")
+
+ cfg = parse_key_value(stdout)
+ assert(cfg['cmp_mode'] == '3')
+ assert(cfg['samples'] == '0')
+ assert(cfg['buffer_length'] == '0')
+ assert(cfg['golomb_par'] == '4')
+ assert(cfg['spill'] == '48')
+ assert(cfg['model_value'] == '8')
+ assert(cfg['round'] == '0')
+ if i > 1:
+ assert(cfg['ap1_golomb_par'] == '3')
+ assert(cfg['ap1_spill'] == '35')
+ assert(cfg['ap2_golomb_par'] == '5')
+ assert(cfg['ap2_spill'] == '60')
+ assert(cfg['rdcu_data_adr'] == '0x000000')
+ assert(cfg['rdcu_model_adr'] == '0x200000')
+ assert(cfg['rdcu_new_model_adr'] == '0x400000')
+ assert(cfg['rdcu_buffer_adr'] == '0x600000')
+
+
+def test_no_data_file_name():
+ returncode, stdout, stderr = call_cmp_tool("-d no_data.data")
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == CMP_START_STR)
+ assert(stderr == "cmp_tool: No configuration file (-c option) or decompression information file (-i option) specified.\n")
+
+
+def test_no_c_or_i_file():
+ returncode, stdout, stderr = call_cmp_tool("-c no_cfg.cfg")
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == CMP_START_STR)
+ assert(stderr == "cmp_tool: No data file (-d option) specified.\n")
+
+
+def test_no_cfg_exist_test():
+ returncode, stdout, stderr = call_cmp_tool("-d no_data.dat -c no_cfg.cfg")
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == CMP_START_STR +
+ "Importing configuration file no_cfg.cfg ... FAILED\n")
+ assert(stderr == "cmp_tool: no_cfg.cfg: No such file or directory\n")
+
+
+def test_compression_diff():
+ # generate test data
+ data = '00 01 00 02 00 03 00 04 00 05 \n'
+ data_file_name = 'data.dat'
+ cfg_file_name = 'diff.cfg'
+ output_prefix = 'diff_cmp'
+ try:
+ with open(data_file_name, 'w', encoding='utf-8') as f:
+ f.write(data)
+
+ # generate test configuration
+ with open(cfg_file_name, 'w', encoding='utf-8') as f:
+ returncode, stdout, stderr = call_cmp_tool("--diff_cfg")
+ assert(returncode == EXIT_SUCCESS)
+ assert(stderr == "")
+ f.write(stdout)
+
+ # compression
+ returncode, stdout, stderr = call_cmp_tool(
+ " -c "+cfg_file_name+" -d "+data_file_name + " -o "+output_prefix)
+
+ # check compression results
+ assert(returncode == EXIT_SUCCESS)
+ assert(stderr == "")
+ assert(stdout == CMP_START_STR +
+ "Importing configuration file %s ... DONE\n" % (cfg_file_name) +
+ "Importing data file %s ... \n" % (data_file_name) +
+ "No samples parameter set. Use samples = 5.\n" +
+ "... DONE\n" +
+ "No buffer_length parameter set. Use buffer_length = 12 as compression buffer size.\n" +
+ "Compress data ... DONE\n" +
+ "Write compressed data to file %s.cmp ... DONE\n" % (output_prefix) +
+ "Write decompression information to file %s.info ... DONE\n" % (output_prefix))
+ # check compressed data
+ with open(output_prefix+".cmp", encoding='utf-8') as f:
+ assert(f.read() == "44 44 40 00 \n")
+ # check info file
+ with open(output_prefix+".info", encoding='utf-8') as f:
+ info = parse_key_value(f.read())
+ assert(info['cmp_mode_used'] == '2')
+ # assert(info['model_value_used'] == '8') # not needed for diff
+ assert(info['round_used'] == '0')
+ assert(info['spill_used'] == '60')
+ assert(info['golomb_par_used'] == '7')
+ assert(info['samples_used'] == '5')
+ assert(info['cmp_size'] == '20')
+ assert(info['cmp_err'] == '0')
+
+ # decompression
+ returncode, stdout, stderr = call_cmp_tool(
+ " -i "+output_prefix+".info -d "+output_prefix+".cmp -o "+output_prefix)
+
+ # check decompression
+ assert(returncode == EXIT_SUCCESS)
+ assert(stdout == CMP_START_STR +
+ "Importing decompression information file %s.info ... DONE\n" % (output_prefix) +
+ "Importing compressed data file %s.cmp ... DONE\n" % (output_prefix) +
+ "Decompress data ... DONE\n" +
+ "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix))
+ assert(stderr == "")
+ with open(output_prefix+".dat", encoding='utf-8') as f:
+ assert(f.read() == data) # decompressed data == input data
+
+ # clean up
+ finally:
+ del_file(data_file_name)
+ del_file(cfg_file_name)
+ del_file(output_prefix+'.cmp')
+ del_file(output_prefix+'.info')
+ del_file(output_prefix+'.dat')
+
+
+def test_model_compression():
+ # generate test data
+ data = '00 01 00 02 00 03 00 04 00 05 \n'
+ model = '00 00 00 01 00 02 00 03 00 04 \n'
+ data_file_name = 'data.dat'
+ model_file_name = 'model.dat'
+ cfg_file_name = 'model.cfg'
+ output_prefix1 = 'model_cmp'
+ output_prefix2 = 'model_decmp'
+ try:
+ with open(data_file_name, 'w', encoding='utf-8') as f:
+ f.write(data)
+ with open(model_file_name, 'w', encoding='utf-8') as f:
+ f.write(model)
+
+ # generate test configuration
+ with open(cfg_file_name, 'w', encoding='utf-8') as f:
+ returncode, stdout, stderr = call_cmp_tool("--model_cfg")
+ assert(returncode == EXIT_SUCCESS)
+ assert(stderr == "")
+ cfg = parse_key_value(stdout)
+ cfg['cmp_mode'] = 'MODE_MODEL_MULTI'
+ cfg["samples"] = '5'
+ cfg["buffer_length"] = '2'
+ for key, value in cfg.items():
+ f.write(key + ' = ' + str(value) + '\n')
+
+ # compression
+ returncode, stdout, stderr = call_cmp_tool(
+ " -c "+cfg_file_name+" -d "+data_file_name + " -m "+model_file_name+" -o "+output_prefix1)
+
+ # check compression results
+ assert(returncode == EXIT_SUCCESS)
+ assert(stderr == "")
+ assert(stdout == CMP_START_STR +
+ "Importing configuration file %s ... DONE\n" % (cfg_file_name) +
+ "Importing data file %s ... DONE\n" % (data_file_name) +
+ "Importing model file %s ... DONE\n" % (model_file_name) +
+ "Compress data ... DONE\n" +
+ "Write compressed data to file %s.cmp ... DONE\n" % (output_prefix1) +
+ "Write decompression information to file %s.info ... DONE\n" % (output_prefix1) +
+ "Write updated model to file %s_upmodel.dat ... DONE\n" % (output_prefix1))
+ # check compressed data
+ with open(output_prefix1+".cmp", encoding='utf-8') as f:
+ assert(f.read() == "49 24 00 00 \n")
+ # check info file
+ with open(output_prefix1+".info", encoding='utf-8') as f:
+ info = parse_key_value(f.read())
+ assert(info['cmp_mode_used'] == '3')
+ assert(info['model_value_used'] == cfg['model_value'])
+ assert(info['round_used'] == cfg['round'])
+ assert(info['spill_used'] == cfg['spill'])
+ assert(info['golomb_par_used'] == cfg['golomb_par'])
+ assert(info['samples_used'] == cfg['samples'])
+ assert(info['cmp_size'] == '15')
+ assert(info['cmp_err'] == '0')
+
+ # decompression
+ returncode, stdout, stderr = call_cmp_tool(
+ " -i "+output_prefix1+".info -d "+output_prefix1+".cmp -m "+model_file_name+" -o "+output_prefix2)
+ assert(returncode == EXIT_SUCCESS)
+ assert(stdout == CMP_START_STR +
+ "Importing decompression information file %s.info ... DONE\n" % (output_prefix1) +
+ "Importing compressed data file %s.cmp ... DONE\n" % (output_prefix1) +
+ "Importing model file %s ... DONE\n" % (model_file_name) +
+ "Decompress data ... DONE\n" +
+ "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix2) +
+ "Write updated model to file %s_upmodel.dat ... DONE\n" % (output_prefix2))
+ assert(stderr == "")
+ # check compressed data
+
+ with open(output_prefix2+".dat", encoding='utf-8') as f:
+ assert(f.read() == data)
+
+ with open(output_prefix1+"_upmodel.dat", encoding='utf-8') as f1:
+ with open(output_prefix2+"_upmodel.dat", encoding='utf-8') as f2:
+ assert(f1.read() == f2.read() ==
+ '00 00 00 01 00 02 00 03 00 04 \n')
+ # clean up
+ finally:
+ del_file(data_file_name)
+ del_file(model_file_name)
+ del_file(cfg_file_name)
+ del_file(output_prefix1+'.cmp')
+ del_file(output_prefix1+'.info')
+ del_file(output_prefix1+'_upmodel.dat')
+ del_file(output_prefix2+'.dat')
+ del_file(output_prefix2+'_upmodel.dat')
+
+
+def test_guess_option():
+ # generate test data
+ data = '00 01 00 01 00 01 00 01 00 01 \n'
+ model = '00 00 00 00 00 00 00 00 00 00 \n'
+ data_file_name = 'data.dat'
+ model_file_name = 'model.dat'
+ args = [
+ ['guess_RDCU_diff', '--guess RDCU -d %s -o guess --rdcu_par' %
+ (data_file_name)],
+ ['guess_RDCU_model', '--guess RDCU -d %s -m %s -o guess --rdcu_par' %
+ (data_file_name, model_file_name)],
+ ['guess_level_3', '--guess MODE_DIFF_MULTI --guess_level 3 -d %s -o guess' %
+ (data_file_name)],
+ ['cfg_folder_not_exist', '--guess RDCU -d ' +
+ data_file_name+' -o not_exist/guess'],
+ ['guess_level_10', '--guess MODE_DIFF_MULTI --guess_level 10 -d %s -o guess' %
+ (data_file_name)],
+ ['guess_unknown_mode', '--guess MODE_UNKNOWN -d %s -o guess' %
+ (data_file_name)],
+ ['guess_model_mode_no_model ', '--guess 1 -d %s -o guess' %
+ (data_file_name)],
+ ]
+ try:
+ with open(data_file_name, 'w', encoding='utf-8') as f:
+ f.write(data)
+ with open(model_file_name, 'w', encoding='utf-8') as f:
+ f.write(model)
+
+ for sub_test, arg in args:
+ returncode, stdout, stderr = call_cmp_tool(arg)
+ if sub_test == 'guess_RDCU_diff' or sub_test == 'guess_RDCU_model' or sub_test == 'guess_level_3':
+ assert(stderr == "")
+ assert(returncode == EXIT_SUCCESS)
+
+ if sub_test == 'guess_RDCU_diff':
+ exp_out = ('', '2', '', '7.27')
+ elif sub_test == 'guess_RDCU_model':
+ exp_out = (
+ 'Importing model file model.dat ... DONE\n', '2', '', '5.33')
+ elif sub_test == 'guess_level_3':
+ exp_out = (
+ '', '3', ' 0%... 6%... 13%... 19%... 25%... 32%... 38%... 44%... 50%... 57%... 64%... 72%... 80%... 88%... 94%... 100%', '11.43')
+ else:
+ exp_out = ('', '', '')
+
+ assert(stdout == CMP_START_STR +
+ "Importing data file %s ... \n" % (data_file_name) +
+ "No samples parameter set. Use samples = 5.\n"
+ "... DONE\n"
+ "%s"
+ "Search for a good set of compression parameters (level: %s) ...%s DONE\n"
+ "Write the guessed compression configuration to file guess.cfg ... DONE\n"
+ "Guessed parameters can compress the data with a CR of %s.\n" % exp_out)
+
+ # check configuration file
+ with open('guess.cfg') as f:
+ cfg = parse_key_value(f.read())
+ assert(cfg['samples'] == '5')
+ assert(cfg['buffer_length'] == '2')
+ assert(cfg['model_value'] == '11')
+ assert(cfg['round'] == '0')
+ if sub_test == 'guess_RDCU_diff':
+ assert(cfg['cmp_mode'] == '2')
+ assert(cfg['golomb_par'] == '2')
+ assert(cfg['spill'] == '22')
+ assert(cfg['ap1_golomb_par'] == '1')
+ assert(cfg['ap1_spill'] == '8')
+ assert(cfg['ap2_golomb_par'] == '3')
+ assert(cfg['ap2_spill'] == '35')
+ assert(cfg['rdcu_data_adr'] == '0x000000')
+ assert(cfg['rdcu_model_adr'] == '0x000000')
+ assert(cfg['rdcu_new_model_adr'] == '0x000000')
+ assert(cfg['rdcu_buffer_adr'] == '0x600000')
+ elif sub_test == 'guess_RDCU_model':
+ assert(cfg['cmp_mode'] == '3')
+ assert(cfg['golomb_par'] == '1')
+ assert(cfg['spill'] == '8')
+ assert(cfg['ap1_golomb_par'] == '2')
+ assert(cfg['ap1_spill'] == '16')
+ assert(cfg['ap2_golomb_par'] == '3')
+ assert(cfg['ap2_spill'] == '23')
+ assert(cfg['rdcu_data_adr'] == '0x000000')
+ assert(cfg['rdcu_model_adr'] == '0x200000')
+ assert(cfg['rdcu_new_model_adr'] == '0x400000')
+ assert(cfg['rdcu_buffer_adr'] == '0x600000')
+ else:
+ assert(cfg['cmp_mode'] == '4')
+ assert(cfg['golomb_par'] == '1')
+ assert(cfg['spill'] == '3')
+ elif sub_test == 'cfg_folder_not_exist':
+ # error case cfg directory does not exit
+ assert(
+ stderr == "cmp_tool: not_exist/guess.cfg: No such file or directory\n")
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == CMP_START_STR +
+ "Importing data file %s ... \n" % (data_file_name) +
+ "No samples parameter set. Use samples = 5.\n" +
+ "... DONE\n"
+ "Search for a good set of compression parameters (level: 2) ... DONE\n" +
+ "Write the guessed compression configuration to file not_exist/guess.cfg ... FAILED\n")
+ elif sub_test == 'guess_level_10':
+ assert(stderr == "cmp_tool: guess level not supported!\n")
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == CMP_START_STR +
+ "Importing data file %s ... \n" % (data_file_name) +
+ "No samples parameter set. Use samples = 5.\n" +
+ "... DONE\n"
+ "Search for a good set of compression parameters (level: 10) ... FAILED\n")
+ elif sub_test == 'guess_unknown_mode':
+ assert(
+ stderr == "cmp_tool: Error: unknown compression mode: MODE_UNKNOWN\n")
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == CMP_START_STR +
+ "Importing data file %s ... \n" % (data_file_name) +
+ "No samples parameter set. Use samples = 5.\n" +
+ "... DONE\n"
+ "Search for a good set of compression parameters (level: 2) ... FAILED\n")
+ else:
+ assert(
+ stderr == "cmp_tool: Error: model mode needs model data (-m option)\n")
+ assert(returncode == EXIT_FAILURE)
+ assert(stdout == CMP_START_STR +
+ "Importing data file %s ... \n" % (data_file_name) +
+ "No samples parameter set. Use samples = 5.\n" +
+ "... DONE\n"
+ "Search for a good set of compression parameters (level: 2) ... FAILED\n")
+ # clean up
+ finally:
+ del_file(data_file_name)
+ del_file(model_file_name)
+ del_file('guess.cfg')
+
+
+def test_small_buf_err():
+ data = '00 01 0F 02 00 0C 00 42 00 23 \n'
+ data_file_name = 'small_buf.dat'
+ cfg_file_name = 'small_buf.cfg'
+ arg = '-c %s -d %s' % (cfg_file_name, data_file_name)
+
+ try:
+ with open(data_file_name, 'w') as f:
+ f.write(data)
+
+ # create a compression file
+ with open(cfg_file_name, 'w') as f:
+ returncode, stdout, stderr = call_cmp_tool("--diff_cfg")
+ assert(returncode == EXIT_SUCCESS)
+ assert(stderr == "")
+ cfg = parse_key_value(stdout)
+ cfg["samples"] = '5'
+ cfg["buffer_length"] = '2'
+ for key, value in cfg.items():
+ f.write(key + ' = ' + str(value) + '\n')
+
+ returncode, stdout, stderr = call_cmp_tool(arg)
+ assert(stdout == CMP_START_STR +
+ "Importing configuration file %s ... DONE\n" % (cfg_file_name) +
+ "Importing data file %s ... DONE\n" % (data_file_name) +
+ "Compress data ... \n"
+ "Compression error 0x01\n"
+ "... FAILED\n")
+ assert(stderr == "cmp_tool: the buffer for the compressed data is too small. Try a larger buffer_length parameter.\n")
+ assert(returncode == EXIT_FAILURE)
+
+ finally:
+ del_file(data_file_name)
+ del_file(cfg_file_name)
+
+# TODO: random test
diff --git a/test/cmp_tool/cmp_tool_lib_test.c b/test/cmp_tool/cmp_tool_lib_test.c
new file mode 100644
index 0000000000000000000000000000000000000000..d55a715acc64053c16c28d585b90cf9387b17375
--- /dev/null
+++ b/test/cmp_tool/cmp_tool_lib_test.c
@@ -0,0 +1,540 @@
+/**
+ * @file test_cmp_tool_lib.c
+ * @author Dominik Loidolt (dominik.loidolt@univie.ac.at)
+ * @date 2020
+ *
+ * @copyright GPLv2
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * @brief unit tests for cmp_tool_lib.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <CUnit/CUnit.h>
+#include <CUnit/TestDB.h>
+#include <CUnit/Basic.h>
+#include <CUnit/Console.h>
+
+#include "../../include/cmp_tool_lib.h"
+#include "../../include/cmp_guess.h"
+
+#define PROGRAM_NAME "cmp_tool"
+
+/* used to redirect stdout to file */
+int fd;
+
+/* used to redirect stdout to file */
+fpos_t pos;
+
+/* @brief The init cmp_tool test suite cleanup functionr. Redirect stdout to
+ * file
+ * @see: http://c-faq.com/stdio/undofreopen.html
+ * @returns zero on success, non-zero otherwise.
+ */
+static int init_cmp_tool(void)
+{
+ fflush(stderr);
+ fgetpos(stderr, &pos);
+ fd = dup(fileno(stderr));
+ if (freopen("tmp_stderr.log", "w", stderr) == NULL) {
+ perror("freopen() failed");
+ return -1;
+ }
+
+ return 0;
+}
+
+
+/* @brief The cmp_tool test suite cleanup function. Closes the temporary file
+ * used by the tests.
+ * @returns zero on success, non-zero otherwise.
+ */
+static int clean_cmp_tool(void)
+{
+ fflush(stderr);
+ dup2(fd, fileno(stderr));
+ close(fd);
+ clearerr(stderr);
+ fsetpos(stderr, &pos); /* for C9X */
+
+ return 0;
+}
+
+
+/* returnd memory must be freed after use */
+char *read_std_err_log(void)
+{
+ char *response = NULL;
+ size_t len = 0;
+ FILE *fp = fopen("tmp_stderr.log", "r");
+ static int n_lines_read =0;
+ int i;
+
+ if (!fp) {
+ puts("File opening failed");
+ abort();
+ }
+
+ fflush(stderr);
+ for (i=0; getline(&response, &len, fp) != -1; i++) {
+ if (i == n_lines_read) {
+ n_lines_read++;
+ /* printf("%zu %s\n",len, response); */
+ fclose(fp);
+ return response;
+ } else {
+ free(response);
+ response = NULL;
+ }
+ }
+ free(response);
+ response = malloc(1);
+ if (!response)
+ abort();
+ response[0] = '\0';
+
+ fclose(fp);
+ return response;
+}
+
+
+/* tests */
+
+/**
+ * @test cmp_tool
+ */
+
+void test_read_file8(void)
+{
+ void *file_name;
+ uint8_t *buf;
+ uint32_t n_word;
+ ssize_t i, size;
+ uint8_t array[33] = {0};
+ char *s;
+
+ /* read in a normal data file */
+ memset(array, 0, sizeof(array));
+ file_name = "test_read_file8_1.txt";
+ buf = array;
+ n_word = 33;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, 33);
+ /* no output on stderr */
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "");
+ free(s);
+
+ /* tests counting the size of a file */
+ file_name = "test_read_file8_2.txt";
+ buf = NULL;
+ n_word = 0;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, 33);
+ /* no output on stderr */
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "");
+ free(s);
+
+ /* test data read in counting comments */
+ file_name = "test_read_file8_2.txt";
+ buf = array;
+ n_word = size;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, 33);
+ for (i = 0; i < size; ++i) {
+ CU_ASSERT_EQUAL(buf[i], i);
+ }
+ /* no output on stderr */
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "");
+ free(s);
+
+ /* tests partially read in */
+ memset(array, 0, sizeof(array));
+ file_name = "test_read_file8_2.txt";
+ buf = array;
+ n_word = 32;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, 32);
+ for (i = 0; i < 33; ++i) {
+ if (i < 32) {
+ CU_ASSERT_EQUAL(buf[i], i);
+ } else {
+ CU_ASSERT_EQUAL(buf[i], 0);
+ }
+ }
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "cmp_tool: test_read_file8_2.txt: Warning: The file may contain more data than specified by the samples or cmp_size parameter.\n");
+ free(s);
+
+ /* tests read 0 words in */
+ memset(array, 0, sizeof(array));
+ file_name = "test_read_file8_2.txt";
+ buf = array;
+ n_word = 0;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, 0);
+ for (i = 0; i < 33; ++i) {
+ CU_ASSERT_EQUAL(buf[i], 0);
+ }
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "cmp_tool: test_read_file8_2.txt: Warning: The file may contain more data than specified by the samples or cmp_size parameter.\n");
+ free(s);
+
+ /* TODO; tests read in 0 empty file */
+
+ /* Error cases */
+ /***************/
+
+ /* file does not contain enough data */
+ memset(array, 0xFF, sizeof(array));
+ file_name = "test_read_file8_2.txt";
+ buf = array;
+ n_word = 34;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, -1);
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "cmp_tool: test_read_file8_2.txt: Error: The files does not contain enough data as given by the n_word parameter.\n");
+ free(s);
+ memset(array, 0xFF, sizeof(array));
+
+ /* no file_name */
+ file_name = NULL;
+ buf = array;
+ n_word = 33;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, -1);
+ /* no output on stderr */
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "");
+ free(s);
+
+ /* wrong file_name */
+ file_name = "file_not_exist.txt";
+ buf = array;
+ n_word = 33;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, -1);
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "cmp_tool: file_not_exist.txt: No such file or directory\n");
+ free(s);
+
+ /* wrong data format part 1/2 */
+ file_name = "test_read_file8_3.txt";
+ buf = array;
+ n_word = 33;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, -1);
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "cmp_tool: test_read_file8_3.txt: Error: The data are not correct formatted. Expected format is like: 12 AB 23 CD .. ..\n");
+ free(s);
+
+ /* wrong data format part 2/2 */
+ file_name = "test_read_file8_4.txt";
+ buf = array;
+ n_word = 33;
+ size = read_file8(file_name, buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, -1);
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "cmp_tool: test_read_file8_4.txt: Error: The data are not correct formatted. Expected format is like: 12 AB 23 CD .. ..\n");
+ free(s);
+
+ /* file error */
+ /* TODO: how to trigger a file error indicator error */
+ if (1) {
+ /* static char fname[] = "test_read_file8_5.txt"; */
+ /* FILE *f = fopen(fname, "wb"); */
+ /* fputs("\xff\xff\n", f); // not a valid UTF-8 character sequence */
+ /* fclose(f); */
+ buf = array;
+ n_word = 33;
+ size = read_file8("test_read_file8_5.txt", buf, n_word, 0);
+ CU_ASSERT_EQUAL(size, -1);
+ CU_ASSERT_STRING_EQUAL(s=read_std_err_log(), "cmp_tool: test_read_file8_5.txt: Error: File error indicator set.\n");
+ free(s);
+ }
+}
+
+
+#define CMP_TEST_SAMPLES 5
+void test_cmp_guess(void)
+{
+ struct cmp_cfg cfg = {0};
+ uint16_t data[CMP_TEST_SAMPLES] = {2,4,6,8,10};
+ uint16_t data_exp[CMP_TEST_SAMPLES] = {2,4,6,8,10};
+ uint16_t model[CMP_TEST_SAMPLES] = {4,6,8,10,12};
+ uint16_t model_exp[CMP_TEST_SAMPLES] = {4,6,8,10,12};
+ uint32_t cmp_size;
+ int level;
+
+
+ /* test 1d-diff mode */
+ cfg.input_buf = data;
+ cfg.model_buf = NULL;
+ cfg.samples = CMP_TEST_SAMPLES;
+ cfg.cmp_mode = MODE_DIFF_MULTI;
+ level = 2;
+ cmp_size = cmp_guess(&cfg, level);
+ CU_ASSERT_TRUE(cmp_size);
+
+ CU_ASSERT_EQUAL(cmp_size, 25);
+ CU_ASSERT_EQUAL(cfg.cmp_mode, MODE_DIFF_MULTI);
+ CU_ASSERT_EQUAL(cfg.golomb_par, 1);
+ CU_ASSERT_EQUAL(cfg.spill, 2);
+ /* CU_ASSERT_EQUAL(cfg.model_value, ); model_value is not needed */
+ CU_ASSERT_EQUAL(cfg.round, 0);
+ CU_ASSERT_EQUAL(cfg.ap1_golomb_par, 2);
+ CU_ASSERT_EQUAL(cfg.ap1_spill, 2);
+ CU_ASSERT_EQUAL(cfg.ap2_golomb_par, 3);
+ CU_ASSERT_EQUAL(cfg.ap2_spill, 2);
+ CU_ASSERT_NSTRING_EQUAL(cfg.input_buf, data_exp, CMP_TEST_SAMPLES);
+ /* CU_ASSERT_NSTRING_EQUAL(cfg.model_buf, model_exp, CMP_TEST_SAMPLES); model is
+ * not used*/
+ CU_ASSERT_EQUAL(cfg.samples, CMP_TEST_SAMPLES);
+ CU_ASSERT_EQUAL(cfg.buffer_length, 2);
+
+
+ /* test model mode */
+ memset(&cfg, 0, sizeof(struct cmp_cfg));
+ cfg.input_buf = data;
+ cfg.model_buf = model;
+ cfg.samples = CMP_TEST_SAMPLES;
+ cfg.cmp_mode = MODE_MODEL_ZERO;
+ level =3;
+
+ cmp_guess_set_model_updates(12);
+ cmp_size = cmp_guess(&cfg, level);
+ CU_ASSERT_TRUE(cmp_size);
+
+ CU_ASSERT_EQUAL(cmp_size, 20);
+ CU_ASSERT_EQUAL(cfg.cmp_mode, 1);
+ CU_ASSERT_EQUAL(cfg.golomb_par, 2);
+ CU_ASSERT_EQUAL(cfg.spill, 22);
+ CU_ASSERT_EQUAL(cfg.model_value, 12);
+ CU_ASSERT_EQUAL(cfg.round, 0);
+ CU_ASSERT_EQUAL(cfg.ap1_golomb_par, 1);
+ CU_ASSERT_EQUAL(cfg.ap1_spill, 8);
+ CU_ASSERT_EQUAL(cfg.ap2_golomb_par, 3);
+ CU_ASSERT_EQUAL(cfg.ap2_spill, 35);
+ CU_ASSERT_NSTRING_EQUAL(cfg.input_buf, data_exp, CMP_TEST_SAMPLES);
+ CU_ASSERT_NSTRING_EQUAL(cfg.model_buf, model_exp, CMP_TEST_SAMPLES);
+ CU_ASSERT_EQUAL(cfg.samples, CMP_TEST_SAMPLES);
+ CU_ASSERT_EQUAL(cfg.buffer_length, 2);
+
+
+ /* test diff mode without model buffer*/
+ cmp_guess_set_model_updates(CMP_GUESS_N_MODEL_UPDATE_DEF);
+ memset(&cfg, 0, sizeof(struct cmp_cfg));
+ cfg.input_buf = data;
+ cfg.model_buf = model;
+ cfg.samples = CMP_TEST_SAMPLES;
+ cfg.cmp_mode = MODE_DIFF_MULTI;
+ level = 3;
+
+ cmp_size = cmp_guess(&cfg, level);
+ CU_ASSERT_TRUE(cmp_size);
+
+ CU_ASSERT_EQUAL(cmp_size, 20);
+ CU_ASSERT_EQUAL(cfg.cmp_mode, MODE_DIFF_MULTI);
+ CU_ASSERT_EQUAL(cfg.golomb_par, 2);
+ CU_ASSERT_EQUAL(cfg.spill, 5);
+ CU_ASSERT_EQUAL(cfg.model_value, 11);
+ CU_ASSERT_EQUAL(cfg.round, 0);
+ CU_ASSERT_EQUAL(cfg.ap1_golomb_par, 1);
+ CU_ASSERT_EQUAL(cfg.ap1_spill, 2);
+ CU_ASSERT_EQUAL(cfg.ap2_golomb_par, 3);
+ CU_ASSERT_EQUAL(cfg.ap2_spill, 2);
+ CU_ASSERT_NSTRING_EQUAL(cfg.input_buf, data_exp, CMP_TEST_SAMPLES);
+ CU_ASSERT_NSTRING_EQUAL(cfg.model_buf, model_exp, CMP_TEST_SAMPLES);
+ CU_ASSERT_EQUAL(cfg.samples, CMP_TEST_SAMPLES);
+ CU_ASSERT_EQUAL(cfg.buffer_length, 2);
+
+
+ /* error test cfg = NULL */
+ level = 2;
+ cmp_size = cmp_guess(NULL, level);
+ CU_ASSERT_FALSE(cmp_size);
+
+
+ /* error test unknown guess_level */
+ level = 4;
+ cmp_size = cmp_guess(&cfg, level);
+ CU_ASSERT_FALSE(cmp_size);
+
+
+ /* error test model mode without model buffer*/
+ memset(&cfg, 0, sizeof(struct cmp_cfg));
+ cfg.input_buf = NULL;
+ cfg.model_buf = NULL;
+ cfg.samples = CMP_TEST_SAMPLES;
+ cfg.cmp_mode = MODE_DIFF_MULTI;
+ level = 2;
+
+ cmp_size = cmp_guess(&cfg, level);
+ CU_ASSERT_FALSE(cmp_size);
+
+
+ /* error test model mode without model buffer*/
+ memset(&cfg, 0, sizeof(struct cmp_cfg));
+ cfg.input_buf = data;
+ cfg.model_buf = NULL;
+ cfg.samples = CMP_TEST_SAMPLES;
+ cfg.cmp_mode = MODE_MODEL_MULTI;
+ level = 2;
+
+ cmp_size = cmp_guess(&cfg, level);
+ CU_ASSERT_FALSE(cmp_size);
+
+
+ /* error test not supported cmp_mode */
+ memset(&cfg, 0, sizeof(struct cmp_cfg));
+ cfg.input_buf = data;
+ cfg.model_buf = model;
+ cfg.samples = CMP_TEST_SAMPLES;
+ cfg.cmp_mode = 5;
+ level = 2;
+
+ cmp_size = cmp_guess(&cfg, level);
+ CU_ASSERT_FALSE(cmp_size);
+
+
+ /* error test samples = 0 */
+ memset(&cfg, 0, sizeof(struct cmp_cfg));
+ cfg.input_buf = data;
+ cfg.model_buf = model;
+ cfg.samples = 0;
+ cfg.cmp_mode = MODE_MODEL_MULTI;
+ level = 2;
+
+ cmp_size = cmp_guess(&cfg, level);
+ CU_ASSERT_FALSE(cmp_size);
+}
+
+void test_cmp_guess_model_value(void)
+{
+ uint16_t model_value;
+
+ model_value = cmp_guess_model_value(0);
+ CU_ASSERT_EQUAL(model_value, 8);
+ model_value = cmp_guess_model_value(1);
+ CU_ASSERT_EQUAL(model_value, 8);
+ model_value = cmp_guess_model_value(2);
+ CU_ASSERT_EQUAL(model_value, 8);
+ model_value = cmp_guess_model_value(3);
+ CU_ASSERT_EQUAL(model_value, 10);
+ model_value = cmp_guess_model_value(4);
+ CU_ASSERT_EQUAL(model_value, 10);
+ model_value = cmp_guess_model_value(5);
+ CU_ASSERT_EQUAL(model_value, 10);
+ model_value = cmp_guess_model_value(6);
+ CU_ASSERT_EQUAL(model_value, 11);
+ model_value = cmp_guess_model_value(10);
+ CU_ASSERT_EQUAL(model_value, 11);
+ model_value = cmp_guess_model_value(11);
+ CU_ASSERT_EQUAL(model_value, 11);
+ model_value = cmp_guess_model_value(12);
+ CU_ASSERT_EQUAL(model_value, 12);
+ model_value = cmp_guess_model_value(20);
+ CU_ASSERT_EQUAL(model_value, 12);
+ model_value = cmp_guess_model_value(21);
+ CU_ASSERT_EQUAL(model_value, 12);
+ model_value = cmp_guess_model_value(22);
+ CU_ASSERT_EQUAL(model_value, 13);
+}
+
+
+void test_cmp_mode_parse(void)
+{
+ uint32_t cmp_mode = ~0;
+ int err;
+
+ /* error cases */
+ err = cmp_mode_parse(NULL, NULL);
+ CU_ASSERT_TRUE(err);
+
+ err = cmp_mode_parse(NULL, &cmp_mode);
+ CU_ASSERT_TRUE(err);
+
+ err = cmp_mode_parse("MODE_RAW", NULL);
+ CU_ASSERT_TRUE(err);
+
+ err = cmp_mode_parse("UNKNOWN", &cmp_mode);
+ CU_ASSERT_TRUE(err);
+
+ err = cmp_mode_parse("9999999999999999999", &cmp_mode);
+ CU_ASSERT_TRUE(err);
+
+ /* mode not defined*/
+ err = cmp_mode_parse("424212", &cmp_mode);
+ CU_ASSERT_TRUE(err);
+
+ /* normal operation */
+ err = cmp_mode_parse("MODE_RAW", &cmp_mode);
+ CU_ASSERT_FALSE(err);
+ CU_ASSERT_EQUAL(cmp_mode, MODE_RAW);
+
+ err = cmp_mode_parse("0", &cmp_mode);
+ CU_ASSERT_FALSE(err);
+ CU_ASSERT_EQUAL(cmp_mode, MODE_RAW);
+
+ err = cmp_mode_parse("0 \n", &cmp_mode);
+ CU_ASSERT_FALSE(err);
+ CU_ASSERT_EQUAL(cmp_mode, MODE_RAW);
+
+ err = cmp_mode_parse(" 2 ", &cmp_mode);
+ CU_ASSERT_FALSE(err);
+ CU_ASSERT_EQUAL(cmp_mode, 2);
+
+ err = cmp_mode_parse("MODE_DIFF_MULTI", &cmp_mode);
+ CU_ASSERT_FALSE(err);
+ CU_ASSERT_EQUAL(cmp_mode, MODE_DIFF_MULTI);
+}
+
+CU_ErrorCode cmp_tool_add_tests(CU_pSuite suite)
+{
+ CU_pTest __attribute__((unused)) test;
+
+ /* add a suite to the registry */
+ suite = CU_add_suite("cmp_tool", init_cmp_tool, clean_cmp_tool);
+ if (suite == NULL) {
+ CU_cleanup_registry();
+ return CU_get_error();
+ }
+
+ /* add the tests to the suite */
+ if ((NULL == CU_add_test(suite, "test of read_file8()", test_read_file8)) ||
+ (NULL == CU_add_test(suite, "test of cmp_mode_parse()", test_cmp_mode_parse))||
+ (NULL == CU_add_test(suite, "test of cmp_guess_model_value()", test_cmp_guess_model_value))||
+ (NULL == CU_add_test(suite, "test of cmp_guess()", test_cmp_guess))) {
+ CU_cleanup_registry();
+ return CU_get_error();
+ }
+
+ return CUE_SUCCESS;
+}
+
+
+#if (__MAIN__)
+int main(void)
+{
+ CU_pSuite suite = NULL;
+
+ /* initialize the CUnit test registry */
+ if (CU_initialize_registry() != CUE_SUCCESS)
+ return CU_get_error();
+
+ cmp_tool_add_tests(suite);
+
+ CU_basic_set_mode(CU_BRM_VERBOSE);
+ CU_basic_run_tests();
+
+ printf("\n\n\n");
+
+ CU_basic_show_failures(CU_get_failure_list());
+
+ CU_cleanup_registry();
+
+ return CU_get_error();
+}
+#endif
diff --git a/test/cmp_tool/test_read_file8_1.txt b/test/cmp_tool/test_read_file8_1.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c700aa60c35b1f2c5b7ca42082d62054511e025d
--- /dev/null
+++ b/test/cmp_tool/test_read_file8_1.txt
@@ -0,0 +1,2 @@
+00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 10
+11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F 20
diff --git a/test/cmp_tool/test_read_file8_2.txt b/test/cmp_tool/test_read_file8_2.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d8122a861aad099273fb76ab6a8e3d87ba8b80c5
--- /dev/null
+++ b/test/cmp_tool/test_read_file8_2.txt
@@ -0,0 +1,10 @@
+# test file to the read_file8 function
+00
+01 02
+03 04 05
+06 07 08 09
+
+0A 0B 0C 0D 0E
+0F 10 11 12 13 14
+# skip this line!
+15 16 17 18 19 1A 1B 1C 1D 1E 1F 20
diff --git a/test/cmp_tool/test_read_file8_3.txt b/test/cmp_tool/test_read_file8_3.txt
new file mode 100644
index 0000000000000000000000000000000000000000..808a543aedc2ce35ebe1fa3726c5a582f1385431
--- /dev/null
+++ b/test/cmp_tool/test_read_file8_3.txt
@@ -0,0 +1,9 @@
+# 2nd test file to the read_file8 function
+# to test the error case when data are wrong encoded
+00 01 02 03 04 05
+06 07 08 09 0A 0B
+0C 0D 0E 0F 10 11
+12 13 14 15 16 17
+18 19 1A 1B 1C 1D
+1E1F 20
+#^ this is the wrong formating
diff --git a/test/cmp_tool/test_read_file8_4.txt b/test/cmp_tool/test_read_file8_4.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8dcdd2a5818bafc3ffb5c8a93cb511b8920cde9a
--- /dev/null
+++ b/test/cmp_tool/test_read_file8_4.txt
@@ -0,0 +1,9 @@
+# 2nd test file to the read_file8 function
+# to test the error case when data are wrong encoded
+00 01 02 03 04 05
+06 07 08 09 0A 0B
+0C 0D 0E 0F 10 11
+12 13 14 15 16 17
+18 19 1A 1B 1C 1D
+1L 1F 20
+#^ this is the wrong formating
diff --git a/test/cmp_tool/test_read_file8_5.txt b/test/cmp_tool/test_read_file8_5.txt
new file mode 100644
index 0000000000000000000000000000000000000000..eac4d84af102ae56ad7c9d905f50db8f2fe5cc3a
--- /dev/null
+++ b/test/cmp_tool/test_read_file8_5.txt
@@ -0,0 +1 @@
+��