From 410511e06d08e84ed79370f837eb1d6194fdd025 Mon Sep 17 00:00:00 2001
From: Dominik Loidolt <dominik.loidolt@univie.ac.at>
Date: Mon, 11 Mar 2024 14:28:48 +0100
Subject: [PATCH] Add chunk compression to cmp_tool
---
meson.build | 2 +-
programs/cmp_io.c | 176 ++++++++++++++++++--------------------------
programs/cmp_io.h | 3 +-
programs/cmp_tool.c | 92 +++++++++++++++++++----
4 files changed, 151 insertions(+), 122 deletions(-)
diff --git a/meson.build b/meson.build
index 6eadf1a..86680ee 100644
--- a/meson.build
+++ b/meson.build
@@ -1,5 +1,5 @@
project('cmp_tool', 'c',
- version : '0.12-b2',
+ version : '0.12-b3',
meson_version : '>= 0.56',
license : 'GPL-2.0',
default_options : [
diff --git a/programs/cmp_io.c b/programs/cmp_io.c
index 90a933b..fd11008 100644
--- a/programs/cmp_io.c
+++ b/programs/cmp_io.c
@@ -31,6 +31,7 @@
#include "cmp_io.h"
#include <cmp_support.h>
+#include <cmp_chunk.h>
#include <rdcu_cmd.h>
#include <byteorder.h>
#include <cmp_data_types.h>
@@ -65,6 +66,7 @@ static const struct {
{DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE, "DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE"},
{DATA_TYPE_F_CAM_OFFSET, "DATA_TYPE_F_CAM_OFFSET"},
{DATA_TYPE_F_CAM_BACKGROUND, "DATA_TYPE_F_CAM_BACKGROUND"},
+ {DATA_TYPE_CHUNK, "DATA_TYPE_CHUNK"},
{DATA_TYPE_UNKNOWN, "DATA_TYPE_UNKNOWN"}
};
@@ -528,22 +530,31 @@ int cmp_mode_parse(const char *cmp_mode_str, enum cmp_mode *cmp_mode)
* @param fp FILE pointer
* @param cfg compression configuration structure holding the read in
* configuration
+ * @param par chunk compression parameters structure holding the read in
+ * chunk configuration
*
* @returns 0 on success, error otherwise
*/
-static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
+static int parse_cfg(FILE *fp, struct cmp_cfg *cfg, struct cmp_par *par)
{
+#define chunk_parse_uint32_parmter(parmter) \
+ if (!strcmp(token1, #parmter)) { \
+ if (atoui32(token1, token2, &par->parmter)) \
+ return -1; \
+ cfg->data_type = DATA_TYPE_CHUNK; \
+ continue; \
+}
char *token1, *token2;
char line[MAX_CONFIG_LINE];
enum {CMP_MODE, SAMPLES, BUFFER_LENGTH, LAST_ITEM};
int j, must_read_items[LAST_ITEM] = {0};
if (!fp)
- return -1;
+ abort();
if (!cfg)
- return -1;
+ abort();
while (fgets(line, sizeof(line), fp) != NULL) {
if (line[0] == '#' || line[0] == '\n')
@@ -576,6 +587,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
must_read_items[CMP_MODE] = 1;
if (cmp_mode_parse(token2, &cfg->cmp_mode))
return -1;
+ par->cmp_mode = cfg->cmp_mode;
continue;
}
if (!strcmp(token1, "golomb_par")) {
@@ -591,6 +603,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
if (!strcmp(token1, "model_value")) {
if (atoui32(token1, token2, &cfg->model_value))
return -1;
+ par->model_value = cfg->model_value;
continue;
}
if (!strcmp(token1, "round")) {
@@ -618,98 +631,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
return -1;
continue;
}
- if (!strcmp(token1, "cmp_par_exp_flags")) {
- if (atoui32(token1, token2, &cfg->cmp_par_exp_flags))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_exp_flags")) {
- if (atoui32(token1, token2, &cfg->spill_exp_flags))
- return -1;
- continue;
- }
- if (!strcmp(token1, "cmp_par_fx")) {
- if (atoui32(token1, token2, &cfg->cmp_par_fx))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_fx")) {
- if (atoui32(token1, token2, &cfg->spill_fx))
- return -1;
- continue;
- }
- if (!strcmp(token1, "cmp_par_ncob")) {
- if (atoui32(token1, token2, &cfg->cmp_par_ncob))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_ncob")) {
- if (atoui32(token1, token2, &cfg->spill_ncob))
- return -1;
- continue;
- }
- if (!strcmp(token1, "cmp_par_efx")) {
- if (atoui32(token1, token2, &cfg->cmp_par_efx))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_efx")) {
- if (atoui32(token1, token2, &cfg->spill_efx))
- return -1;
- continue;
- }
- if (!strcmp(token1, "cmp_par_ecob")) {
- if (atoui32(token1, token2, &cfg->cmp_par_ecob))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_ecob")) {
- if (atoui32(token1, token2, &cfg->spill_ecob))
- return -1;
- continue;
- }
- if (!strcmp(token1, "cmp_par_fx_cob_variance")) {
- if (atoui32(token1, token2, &cfg->cmp_par_fx_cob_variance))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_fx_cob_variance")) {
- if (atoui32(token1, token2, &cfg->spill_fx_cob_variance))
- return -1;
- continue;
- }
-#if 0
- if (!strcmp(token1, "cmp_par_mean")) {
- if (atoui32(token1, token2, &cfg->cmp_par_mean))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_mean")) {
- if (atoui32(token1, token2, &cfg->spill_mean))
- return -1;
- continue;
- }
- if (!strcmp(token1, "cmp_par_variance")) {
- if (atoui32(token1, token2, &cfg->cmp_par_variance))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_variance")) {
- if (atoui32(token1, token2, &cfg->spill_variance))
- return -1;
- continue;
- }
- if (!strcmp(token1, "cmp_par_pixels_error")) {
- if (atoui32(token1, token2, &cfg->cmp_par_pixels_error))
- return -1;
- continue;
- }
- if (!strcmp(token1, "spill_pixels_error")) {
- if (atoui32(token1, token2, &cfg->spill_pixels_error))
- return -1;
- continue;
- }
-#endif
+
if (!strcmp(token1, "rdcu_data_adr")) {
int i = sram_addr_to_int(token2);
@@ -766,18 +688,57 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
must_read_items[BUFFER_LENGTH] = 1;
continue;
}
+
+ /* chunk_parse_uint32_parmter(model_value); */
+ chunk_parse_uint32_parmter(lossy_par);
+
+ chunk_parse_uint32_parmter(nc_imagette);
+
+ chunk_parse_uint32_parmter(s_exp_flags);
+ chunk_parse_uint32_parmter(s_fx);
+ chunk_parse_uint32_parmter(s_ncob);
+ chunk_parse_uint32_parmter(s_efx);
+ chunk_parse_uint32_parmter(s_ecob);
+
+ chunk_parse_uint32_parmter(l_exp_flags);
+ chunk_parse_uint32_parmter(l_fx);
+ chunk_parse_uint32_parmter(l_ncob);
+ chunk_parse_uint32_parmter(l_efx);
+ chunk_parse_uint32_parmter(l_ecob);
+ chunk_parse_uint32_parmter(l_fx_cob_variance);
+
+ chunk_parse_uint32_parmter(saturated_imagette);
+
+ chunk_parse_uint32_parmter(nc_offset_mean);
+ chunk_parse_uint32_parmter(nc_offset_variance);
+ chunk_parse_uint32_parmter(nc_background_mean);
+ chunk_parse_uint32_parmter(nc_background_variance);
+ chunk_parse_uint32_parmter(nc_background_outlier_pixels);
+
+ chunk_parse_uint32_parmter(smearing_mean);
+ chunk_parse_uint32_parmter(smearing_variance_mean);
+ chunk_parse_uint32_parmter(smearing_outlier_pixels);
+
+ chunk_parse_uint32_parmter(fc_imagette);
+ chunk_parse_uint32_parmter(fc_offset_mean);
+ chunk_parse_uint32_parmter(fc_offset_variance);
+ chunk_parse_uint32_parmter(fc_background_mean);
+ chunk_parse_uint32_parmter(fc_background_variance);
+ chunk_parse_uint32_parmter(fc_background_outlier_pixels);
}
- if (raw_mode_is_used(cfg->cmp_mode))
- if (must_read_items[CMP_MODE] == 1 &&
- must_read_items[BUFFER_LENGTH] == 1)
- return 0;
+ if (cfg->data_type != DATA_TYPE_CHUNK) {
+ if (raw_mode_is_used(cfg->cmp_mode))
+ if (must_read_items[CMP_MODE] == 1 &&
+ must_read_items[BUFFER_LENGTH] == 1)
+ return 0;
- for (j = 0; j < LAST_ITEM; j++) {
- if (must_read_items[j] < 1) {
- fprintf(stderr, "%s: Some parameters are missing. Check if the following parameters: cmp_mode, golomb_par, spill, samples and buffer_length are all set in the configuration file.\n",
- PROGRAM_NAME);
- return -1;
+ for (j = 0; j < LAST_ITEM; j++) {
+ if (must_read_items[j] < 1) {
+ fprintf(stderr, "%s: Some parameters are missing. Check if the following parameters: cmp_mode, golomb_par, spill, samples and buffer_length are all set in the configuration file.\n",
+ PROGRAM_NAME);
+ return -1;
+ }
}
}
@@ -791,12 +752,15 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
* @param file_name file containing the compression configuration file
* @param cfg compression configuration structure holding the read in
* configuration
+ * @param par chunk compression parameters structure holding the read
+ * in chunk configuration
* @param verbose_en print verbose output if not zero
*
* @returns 0 on success, error otherwise
*/
-int cmp_cfg_read(const char *file_name, struct cmp_cfg *cfg, int verbose_en)
+int cmp_cfg_read(const char *file_name, struct cmp_cfg *cfg, struct cmp_par *par,
+ int verbose_en)
{
int err;
FILE *fp;
@@ -820,7 +784,7 @@ int cmp_cfg_read(const char *file_name, struct cmp_cfg *cfg, int verbose_en)
return -1;
}
- err = parse_cfg(fp, cfg);
+ err = parse_cfg(fp, cfg, par);
fclose(fp);
if (err)
return err;
diff --git a/programs/cmp_io.h b/programs/cmp_io.h
index 60c3910..c1f4e86 100644
--- a/programs/cmp_io.h
+++ b/programs/cmp_io.h
@@ -24,6 +24,7 @@
#include <string.h>
#include <cmp_support.h>
+#include <cmp_chunk.h>
#include <cmp_entity.h>
#define MAX_CONFIG_LINE 256
@@ -39,7 +40,7 @@
void print_help(const char *program_name);
-int cmp_cfg_read(const char *file_name, struct cmp_cfg *cfg, int verbose_en);
+int cmp_cfg_read(const char *file_name, struct cmp_cfg *cfg, struct cmp_par *par, int verbose_en);
int cmp_info_read(const char *file_name, struct cmp_info *info, int verbose_en);
ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t buf_size, int flags);
diff --git a/programs/cmp_tool.c b/programs/cmp_tool.c
index 9560c9d..3dd5a9a 100644
--- a/programs/cmp_tool.c
+++ b/programs/cmp_tool.c
@@ -27,6 +27,7 @@
#include "cmp_tool-config.h"
#include "cmp_io.h"
#include "cmp_icu.h"
+#include "cmp_chunk.h"
#include "cmp_rdcu.h"
#include "decmp.h"
#include "cmp_guess.h"
@@ -48,6 +49,9 @@ static enum cmp_data_type parse_data_type(const char *data_type_str);
static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode,
int guess_level);
+/* compress chunk data and write the results to files */
+static int compression_of_chunk(void *chunk, uint32_t size, void *model, struct cmp_par *chunk_par);
+
/* compress the data and write the results to files */
static int compression(struct cmp_cfg *cfg, struct cmp_info *info);
@@ -152,9 +156,12 @@ int main(int argc, char **argv)
struct cmp_entity *decomp_entity = NULL;
/* buffer containing the read in model */
uint16_t *input_model_buf = NULL;
+ /* size of the data to be compressed and the model of it */
+ uint32_t input_size;
struct cmp_info info = {0}; /* decompression information struct */
struct cmp_cfg cfg = {0}; /* compressor configuration struct */
+ struct cmp_par chunk_par = {0}; /* compressor parameters for chunk compression */
cfg.data_type = DATA_TYPE_IMAGETTE; /* use imagette as default data type */
cfg.max_used_bits = &MAX_USED_BITS_SAFE; /* define max_used_bits default */
@@ -327,12 +334,11 @@ int main(int argc, char **argv)
if (cmp_operation || guess_operation) {
ssize_t size;
- uint32_t input_size;
if (cmp_operation) {
printf("## Starting the compression ##\n");
printf("Importing configuration file %s ... ", cfg_file_name);
- error = cmp_cfg_read(cfg_file_name, &cfg, io_flags & CMP_IO_VERBOSE);
+ error = cmp_cfg_read(cfg_file_name, &cfg, &chunk_par, io_flags & CMP_IO_VERBOSE);
if (error)
goto fail;
printf("DONE\n");
@@ -342,20 +348,28 @@ int main(int argc, char **argv)
printf("Importing data file %s ... ", data_file_name);
/* count the samples in the data file when samples == 0 */
- if (cfg.samples == 0) {
- int32_t samples;
+ if (cfg.data_type != DATA_TYPE_CHUNK) {
+ if (cfg.samples == 0) {
+ int32_t samples;
+
+ size = read_file_data(data_file_name, cfg.data_type, NULL, 0, io_flags);
+ if (size <= 0 || size > UINT32_MAX) /* empty file is treated as an error */
+ goto fail;
+ samples = cmp_input_size_to_samples((uint32_t)size, cfg.data_type);
+ if (samples < 0)
+ goto fail;
+ cfg.samples = (uint32_t)samples;
+ printf("\nNo samples parameter set. Use samples = %u.\n... ", cfg.samples);
+ }
- size = read_file_data(data_file_name, cfg.data_type, NULL, 0, io_flags);
+ input_size = cmp_cal_size_of_data(cfg.samples, cfg.data_type);
+ } else {
+ size = read_file_data(data_file_name, cfg.data_type, NULL, 0, io_flags);
if (size <= 0 || size > UINT32_MAX) /* empty file is treated as an error */
goto fail;
- samples = cmp_input_size_to_samples((uint32_t)size, cfg.data_type);
- if (samples < 0)
- goto fail;
- cfg.samples = (uint32_t)samples;
- printf("\nNo samples parameter set. Use samples = %u.\n... ", cfg.samples);
+ input_size = (uint32_t)size;
}
- input_size = cmp_cal_size_of_data(cfg.samples, cfg.data_type);
cfg.input_buf = malloc(input_size);
if (!cfg.input_buf) {
fprintf(stderr, "%s: Error allocating memory for input data buffer.\n", PROGRAM_NAME);
@@ -456,7 +470,7 @@ int main(int argc, char **argv)
if (cmp_operation || guess_operation) {
data_type = cfg.data_type;
- model_size = cmp_cal_size_of_data(cfg.samples, cfg.data_type);
+ model_size = input_size;
} else {
data_type = cmp_ent_get_data_type(decomp_entity);
model_size = cmp_ent_get_original_size(decomp_entity);
@@ -483,7 +497,11 @@ int main(int argc, char **argv)
if (error)
goto fail;
} else if (cmp_operation) {
- error = compression(&cfg, &info);
+ if (cfg.data_type == DATA_TYPE_CHUNK)
+ error = compression_of_chunk(cfg.input_buf, input_size,
+ input_model_buf, &chunk_par);
+ else
+ error = compression(&cfg, &info);
if (error)
goto fail;
} else {
@@ -502,7 +520,7 @@ int main(int argc, char **argv)
printf("Write updated model to file %s_upmodel.dat ... ", output_prefix);
if (cmp_operation) {
data_type = cfg.data_type;
- model_size = cmp_cal_size_of_data(cfg.samples, data_type);
+ model_size = input_size;
} else {
data_type = cmp_ent_get_data_type(decomp_entity);
model_size = cmp_ent_get_original_size(decomp_entity);
@@ -705,6 +723,52 @@ static int cmp_gernate_rdcu_info(const struct cmp_cfg *cfg, int cmp_size_bit,
}
+/**
+ * @brief compress chunk data and write the results to files
+ */
+
+static int compression_of_chunk(void *chunk, uint32_t size, void *model, struct cmp_par *chunk_par)
+{
+ uint32_t bound = compress_chunk_cmp_size_bound(chunk, size);
+ uint32_t *cmp_data;
+ int32_t cmp_size;
+ int error;
+
+ if (!bound)
+ return -1;
+ cmp_data = calloc(1, bound);
+ if (cmp_data == NULL) {
+ fprintf(stderr, "%s: Error allocating memory for output buffer.\n", PROGRAM_NAME);
+ return -1;
+ }
+
+ printf("Compress chunk data ... ");
+ cmp_size = compress_chunk(chunk, size, model, model,
+ cmp_data, bound, chunk_par);
+
+ if (cmp_size < 0) {
+ if (cmp_size == CMP_ERROR_SMALL_BUF)
+ fprintf(stderr, "Error: The buffer for the compressed data is too small to hold the compressed data. Try a larger buffer_length parameter.\n");
+ free(cmp_data);
+ cmp_data = NULL;
+ printf("FAILED\n");
+ return -1;
+ }
+
+ printf("DONE\nWrite compressed data to file %s.cmp ... ", output_prefix);
+ error = write_data_to_file(cmp_data, (uint32_t)cmp_size, output_prefix,
+ ".cmp", io_flags);
+ free(cmp_data);
+ cmp_data = NULL;
+ if (error) {
+ printf("FAILED\n");
+ return -1;
+ }
+ printf("DONE\n");
+ return 0;
+}
+
+
/**
* @brief compress the data and write the results to files
*/
--
GitLab