Coverage Report

Created: 2025-06-15 00:57

/src/cmp_tool/programs/cmp_tool.c
Line
Count
Source (jump to first uncovered line)
1
/**
2
 * @file   cmp_tool.c
3
 * @author Dominik Loidolt (dominik.loidolt@univie.ac.at)
4
 * @author Johannes Seelig (johannes.seelig@univie.ac.at)
5
 * @date   2020
6
 *
7
 * @copyright GPLv2
8
 * This program is free software; you can redistribute it and/or modify it
9
 * under the terms and conditions of the GNU General Public License,
10
 * version 2, as published by the Free Software Foundation.
11
 *
12
 * This program is distributed in the hope it will be useful, but WITHOUT
13
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15
 * more details.
16
 *
17
 * @brief command line tool for PLATO ICU/RDCU compression/decompression
18
 * @see README.md
19
 * @see Data Compression User Manual PLATO-UVIE-PL-UM-0001
20
 */
21
22
#include <stddef.h>
23
#include <stdint.h>
24
#include <stdio.h>
25
#include <stdlib.h>
26
#include <limits.h>
27
#include <string.h>
28
#include <errno.h>
29
#include <getopt.h>
30
31
#include <cmp_tool-config.h>
32
33
#include "cmp_support.h"
34
#include "cmp_io.h"
35
#include "cmp_icu.h"
36
#include "cmp_chunk.h"
37
#include "cmp_rdcu_cfg.h"
38
#include "decmp.h"
39
#include "cmp_guess.h"
40
#include "cmp_entity.h"
41
#include "rdcu_pkt_to_file.h"
42
43
44
327
#define BUFFER_LENGTH_DEF_FAKTOR 2
45
46
1.75k
#define DEFAULT_MODEL_ID 53264  /* random default id */
47
48
49
/**
50
 * @brief checks if an optional argument is present
51
 *
52
 * this macro evaluates whether the current argument pointer optarg is null and
53
 * if there is a valid argument present at the current index of argv it updates
54
 * optarg and increments the index optind if an argument is found it also
55
 * ensures that the argument is not null empty or another option
56
 *
57
 * @return true if an optional argument is present and updates optarg
58
 * @see https://stackoverflow.com/a/69177115
59
 */
60
61
#define OPTIONAL_ARGUMENT_IS_PRESENT \
62
123
  ((optarg == NULL \
63
123
  && optind < argc /* make sure optind is valid */ \
64
123
  && NULL != argv[optind] /* make sure it's not a null string */ \
65
123
  && '\0' != argv[optind][0] /* ... or an empty string */ \
66
123
  && '-' != argv[optind][0]) /* ... or another option */ \
67
123
  ? ((optarg = argv[optind++]) != NULL) /* update optind so the next getopt_long invocation skips argv[optind] */ \
68
123
  : (optarg != NULL))
69
70
71
/* find a good set of compression parameters for a given dataset */
72
static int guess_cmp_pars(struct rdcu_cfg *rcfg, struct cmp_par *chunk_par,
73
        uint32_t input_size, const char *guess_option, const
74
        char *guess_level_str);
75
76
/* compress chunk data and write the results to files */
77
static int compression_of_chunk(const void *chunk, uint32_t size, void *model,
78
        const struct cmp_par *chunk_par);
79
80
/* compress the data and write the results to files */
81
static int compression_for_rdcu(struct rdcu_cfg *rcfg);
82
83
/* decompress the data and write the results in file(s)*/
84
static int decompression(const struct cmp_entity *ent, uint16_t *input_model_buf);
85
86
/* create a default configuration for a compression data type */
87
enum cfg_default_opt {DIFF_CFG, MODEL_CFG};
88
static void cmp_cfg_create_default(struct rdcu_cfg *rcfg, enum cfg_default_opt mode);
89
90
91
/*
92
 * For long options that have no equivalent short option, use a
93
 * non-character as a pseudo short option, starting with CHAR_MAX + 1.
94
 */
95
enum {
96
  DIFF_CFG_OPTION = CHAR_MAX + 1,
97
  GUESS_OPTION,
98
  GUESS_LEVEL,
99
  RDCU_PKT_OPTION,
100
  LAST_INFO,
101
  NO_HEADER,
102
  MODEL_ID,
103
  MODEL_COUTER
104
};
105
106
static const struct option long_options[] = {
107
  {"rdcu_par", no_argument, NULL, 'a'},
108
  {"model_cfg", no_argument, NULL, 'n'},
109
  {"help", no_argument, NULL, 'h'},
110
  {"verbose", no_argument, NULL, 'v'},
111
  {"version", no_argument, NULL, 'V'},
112
  {"rdcu_pkt", no_argument, NULL, RDCU_PKT_OPTION},
113
  {"diff_cfg", no_argument, NULL, DIFF_CFG_OPTION},
114
  {"guess", optional_argument, NULL, GUESS_OPTION},
115
  {"guess_level", required_argument, NULL, GUESS_LEVEL},
116
  {"last_info", required_argument, NULL, LAST_INFO},
117
  {"no_header", no_argument, NULL, NO_HEADER},
118
  {"model_id", required_argument, NULL, MODEL_ID},
119
  {"model_counter", required_argument, NULL, MODEL_COUTER},
120
  {"binary", no_argument, NULL, 'b'},
121
  {NULL, 0, NULL, 0}
122
};
123
124
/* prefix of the generated output file names */
125
static const char *output_prefix = DEFAULT_OUTPUT_PREFIX;
126
127
/* if non zero additional RDCU parameters are included in the compression
128
 * configuration and decompression information files
129
 */
130
static int add_rdcu_pars;
131
132
/* if non zero generate RDCU setup packets */
133
static int rdcu_pkt_mode;
134
135
/* file name of the last compression information file to generate parallel RDCU setup packets */
136
static const char *last_info_file_name;
137
138
/* option flags for file IO */
139
static int io_flags;
140
141
/* if non zero add a compression entity header in front of the compressed data */
142
static int include_cmp_header = 1;
143
144
/* model ID set by the --model_id option */
145
static uint32_t model_id = DEFAULT_MODEL_ID;
146
147
/* model counter set by the --model_counter option */
148
static uint32_t model_counter;
149
150
151
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
152
1.75k
#define CMP_MAIN cmp_tool_main
153
/* Redefine (f)printf to do nothing */
154
__extension__
155
189k
#define printf(...) do {} while (0)
156
314
#define fprintf(...) do {} while (0)
157
56
#define fflush(...) do {} while (0)
158
159
int CMP_MAIN(int argc, char **argv);
160
161
/**
162
 * @brief callable main function
163
 * Resets the global variables and calls the cmp_tool main function.
164
 *
165
 * @param argc argument count
166
 * @param argv argument vector
167
 *
168
 * @returns EXIT_SUCCESS on success, EXIT_FAILURE on error.
169
 */
170
171
int testable_cmp_tool_main(int argc, char **argv)
172
1.75k
{
173
1.75k
  output_prefix = DEFAULT_OUTPUT_PREFIX;
174
1.75k
  add_rdcu_pars = 0;
175
1.75k
  rdcu_pkt_mode = 0;
176
1.75k
  last_info_file_name = NULL;
177
1.75k
  io_flags = 0;
178
1.75k
  include_cmp_header = 1;
179
1.75k
  model_id = DEFAULT_MODEL_ID;
180
1.75k
  model_counter = 0;
181
182
1.75k
  optind = 0;
183
1.75k
  return CMP_MAIN(argc, argv);
184
1.75k
}
185
#else
186
#define CMP_MAIN main
187
#endif
188
189
190
/**
191
 * @brief This is the main function of the compression / decompression tool
192
 *
193
 * @param argc argument count
194
 * @param argv argument vector
195
 * @see README.md
196
 *
197
 * @returns EXIT_SUCCESS on success, EXIT_FAILURE on error
198
 */
199
200
int CMP_MAIN(int argc, char **argv)
201
1.75k
{
202
1.75k
  int opt;
203
1.75k
  int error;
204
205
1.75k
  const char *cfg_file_name = NULL;
206
1.75k
  const char *info_file_name = NULL;
207
1.75k
  const char *data_file_name = NULL;
208
1.75k
  const char *model_file_name = NULL;
209
1.75k
  const char *guess_option = NULL;
210
1.75k
  const char *guess_level_str = NULL;
211
1.75k
  const char *program_name = argv[0];
212
213
1.75k
  int cmp_operation = 0;
214
1.75k
  int print_model_cfg = 0;
215
1.75k
  int guess_operation = 0;
216
1.75k
  int print_diff_cfg = 0;
217
218
  /* buffer containing all read in compressed data for decompression */
219
1.75k
  struct cmp_entity *decomp_entity = NULL;
220
  /* buffer containing the read in model */
221
1.75k
  uint16_t *input_model_buf = NULL;
222
  /* size of the data to be compressed and the model of it */
223
1.75k
  uint32_t input_size = 0;
224
225
1.75k
  struct cmp_info info = {0}; /* RDCU decompression information struct */
226
1.75k
  struct rdcu_cfg rcfg = {0}; /* RDCU compressor configuration struct */
227
1.75k
  struct cmp_par chunk_par = {0}; /* compressor parameters for chunk compression */
228
1.75k
  enum cmp_type cmp_type;
229
230
  /* show help if no arguments are provided */
231
1.75k
  if (argc < 2) {
232
14
    print_help(program_name);
233
14
    return EXIT_FAILURE;
234
14
  }
235
236
9.08k
  while ((opt = getopt_long(argc, argv, "abc:d:hi:m:no:vV", long_options,
237
9.08k
          NULL)) != -1) {
238
7.37k
    switch (opt) {
239
124
    case 'a': /* --rdcu_par */
240
124
      add_rdcu_pars = 1;
241
124
      break;
242
169
    case 'b':
243
169
      io_flags |= CMP_IO_BINARY;
244
169
      break;
245
1.34k
    case 'c':
246
1.34k
      cmp_operation = 1;
247
1.34k
      cfg_file_name = optarg;
248
1.34k
      break;
249
1.61k
    case 'd':
250
1.61k
      data_file_name = optarg;
251
1.61k
      break;
252
1
    case 'h': /* --help */
253
1
      print_help(argv[0]);
254
1
      return EXIT_SUCCESS;
255
1.04k
    case 'i':
256
1.04k
      info_file_name = optarg;
257
1.04k
      include_cmp_header = 0;
258
1.04k
      break;
259
888
    case 'm': /* read model */
260
888
      model_file_name = optarg;
261
888
      break;
262
89
    case 'n': /* --model_cfg */
263
89
      print_model_cfg = 1;
264
89
      break;
265
1.79k
    case 'o':
266
1.79k
      output_prefix = optarg;
267
1.79k
      break;
268
95
    case 'v': /* --verbose */
269
95
      if (io_flags & CMP_IO_VERBOSE)
270
74
        io_flags |= CMP_IO_VERBOSE_EXTRA;
271
95
      io_flags |= CMP_IO_VERBOSE;
272
95
      break;
273
2
    case 'V': /* --version */
274
2
      printf("%s version %s\n", PROGRAM_NAME, CMP_TOOL_VERSION);
275
2
      return EXIT_SUCCESS;
276
13
    case DIFF_CFG_OPTION:
277
13
      print_diff_cfg = 1;
278
13
      break;
279
123
    case GUESS_OPTION:
280
123
      guess_operation = 1;
281
123
      if (OPTIONAL_ARGUMENT_IS_PRESENT)
282
83
        guess_option = optarg;
283
123
      break;
284
12
    case GUESS_LEVEL:
285
12
      guess_level_str = optarg;
286
12
      break;
287
6
    case LAST_INFO:
288
6
      last_info_file_name = optarg;
289
      /* fall through */
290
13
    case RDCU_PKT_OPTION:
291
13
      rdcu_pkt_mode = 1;
292
13
      add_rdcu_pars = 1;
293
      /* fall through */
294
26
    case NO_HEADER:
295
26
      include_cmp_header = 0;
296
26
      break;
297
12
    case MODEL_ID:
298
12
      if (atoui32("model_id", optarg, &model_id))
299
1
        return EXIT_FAILURE;
300
11
      if (model_counter > UINT16_MAX) {
301
0
        fprintf(stderr, "%s: Error: model id value to large.\n", PROGRAM_NAME);
302
0
        return EXIT_FAILURE;
303
0
      }
304
11
      break;
305
11
    case MODEL_COUTER:
306
5
      if (atoui32("model_counter", optarg, &model_counter))
307
1
        return EXIT_FAILURE;
308
4
      if (model_counter > UINT8_MAX) {
309
1
        fprintf(stderr, "%s: Error: model counter value to large.\n", PROGRAM_NAME);
310
1
        return EXIT_FAILURE;
311
1
      }
312
3
      break;
313
23
    default:
314
23
      print_help(program_name);
315
23
      return EXIT_FAILURE;
316
7.37k
    }
317
7.37k
  }
318
1.71k
  argc -= optind;
319
320
#ifdef ARGUMENT_INPUT_MODE
321
322
  argv += optind;
323
  if (argc > 2) {
324
    printf("%s: To many arguments.\n", PROGRAM_NAME);
325
    print_help(argv[0]);
326
    return EXIT_FAILURE;
327
  }
328
329
  if (argc > 0) {
330
    if (!data_file_name)
331
      data_file_name = argv[0];
332
    else {
333
      printf("You can define the data file using either the -d option or the first argument, but not both.\n");
334
      print_help(program_name);
335
      return EXIT_FAILURE;
336
    }
337
  }
338
  if (argc > 1) {
339
    if (!model_file_name)
340
      model_file_name = argv[1];
341
    else {
342
      printf("You can define the model file using either the -m option or the second argument, but not both.\n");
343
      print_help(program_name);
344
      return EXIT_FAILURE;
345
    }
346
  }
347
#else
348
1.71k
  if (argc > 0) {
349
169
    printf("%s: To many arguments.\n", PROGRAM_NAME);
350
169
    print_help(argv[0]);
351
169
    return EXIT_FAILURE;
352
169
  }
353
1.54k
#endif
354
355
1.54k
  if (print_model_cfg || print_diff_cfg) {
356
20
    if (print_model_cfg && print_diff_cfg) {
357
2
      fprintf(stderr, "%s: Cannot use -n, --model_cfg and -diff_cfg together.\n",
358
2
        PROGRAM_NAME);
359
2
      return EXIT_FAILURE;
360
2
    }
361
18
    if (print_model_cfg)
362
16
      cmp_cfg_create_default(&rcfg, MODEL_CFG);
363
2
    else
364
2
      cmp_cfg_create_default(&rcfg, DIFF_CFG);
365
18
    cmp_cfg_print(&rcfg, add_rdcu_pars);
366
18
    return EXIT_SUCCESS;
367
20
  }
368
369
1.52k
  {
370
1.52k
    static const char str[] = "### PLATO Compression/Decompression Tool Version " CMP_TOOL_VERSION " ###\n";
371
1.52k
    size_t str_len = strlen(str) - 1; /* -1 for \n */
372
1.52k
    size_t i;
373
374
88.4k
    for (i = 0; i < str_len; i++)
375
86.9k
      printf("#");
376
1.52k
    printf("\n");
377
1.52k
    printf("%s", str);
378
88.4k
    for (i = 0; i < str_len; i++)
379
86.9k
      printf("#");
380
1.52k
    printf("\n");
381
1.52k
  }
382
383
1.52k
  if (!data_file_name) {
384
83
    fprintf(stderr, "%s: No data file (-d option) specified.\n", PROGRAM_NAME);
385
83
    return EXIT_FAILURE;
386
83
  }
387
388
1.44k
  if (!cfg_file_name && !info_file_name && !guess_operation && !include_cmp_header) {
389
3
    fprintf(stderr, "%s: No configuration file (-c option) or decompression information file (-i option) specified.\n",
390
3
      PROGRAM_NAME);
391
3
    return EXIT_FAILURE;
392
3
  }
393
394
395
1.43k
  if (cmp_operation || guess_operation) {
396
1.09k
    ssize_t size;
397
398
1.09k
    if (cmp_operation) {
399
1.02k
      printf("## Starting the compression ##\n");
400
1.02k
      printf("Importing configuration file %s ... ", cfg_file_name);
401
1.02k
      cmp_type = cmp_cfg_read(cfg_file_name, &rcfg, &chunk_par, io_flags & CMP_IO_VERBOSE);
402
1.02k
      if (cmp_type == CMP_TYPE_ERROR)
403
420
        goto fail;
404
605
      printf("DONE\n");
405
605
    } else { /* guess_operation */
406
70
      printf("## Search for a good set of compression parameters ##\n");
407
408
70
      if (guess_option == NULL || !case_insensitive_compare(guess_option, "chunk"))
409
9
        cmp_type = CMP_TYPE_CHUNK;
410
61
      else
411
61
        cmp_type = CMP_TYPE_RDCU;
412
70
    }
413
414
675
    printf("Importing data file %s ... ", data_file_name);
415
675
    if (cmp_type == CMP_TYPE_RDCU) {
416
244
      if (rcfg.samples == 0) {
417
        /* count the samples in the data file when samples == 0 */
418
101
        size = read_file_data(data_file_name, cmp_type, NULL, 0, io_flags);
419
101
        if (size <= 0 || size > INT32_MAX || (size_t)size % sizeof(uint16_t)) /* empty file is treated as an error */
420
5
          goto fail;
421
96
        rcfg.samples = (uint32_t)((size_t)size/sizeof(uint16_t));
422
96
        printf("\nNo samples parameter set. Use samples = %u.\n... ", rcfg.samples);
423
96
      }
424
425
239
      input_size = rcfg.samples * sizeof(uint16_t);
426
431
    } else {
427
431
      size  = read_file_data(data_file_name, cmp_type, NULL, 0, io_flags);
428
431
      if (size <= 0 || size > INT32_MAX) /* empty file is treated as an error */
429
51
        goto fail;
430
380
      input_size = (uint32_t)size;
431
380
    }
432
433
619
    if (input_size > CMP_ENTITY_MAX_ORIGINAL_SIZE) {
434
1
      fprintf(stderr, "%s: Error input data size is to large; maximum original data size: %lu\n", PROGRAM_NAME, CMP_ENTITY_MAX_ORIGINAL_SIZE);
435
1
      goto fail;
436
1
    }
437
438
618
    rcfg.input_buf = malloc(input_size);
439
618
    if (!rcfg.input_buf) {
440
0
      fprintf(stderr, "%s: Error allocating memory for input data buffer.\n", PROGRAM_NAME);
441
0
      goto fail;
442
0
    }
443
444
618
    size = read_file_data(data_file_name, cmp_type, rcfg.input_buf,
445
618
              input_size, io_flags);
446
618
    if (size < 0)
447
89
      goto fail;
448
529
    printf("DONE\n");
449
450
529
  } else { /* decompression mode*/
451
344
    printf("## Starting the decompression ##\n");
452
344
    if (info_file_name) {
453
217
      ssize_t f_size;
454
217
      size_t ent_size;
455
456
217
      printf("Importing decompression information file %s ... ", info_file_name);
457
217
      error  = cmp_info_read(info_file_name, &info, io_flags & CMP_IO_VERBOSE);
458
217
      if (error)
459
213
        goto fail;
460
4
      printf("DONE\n");
461
462
4
      printf("Importing compressed data file %s ... ", data_file_name);
463
464
4
      ent_size = cmp_ent_create(NULL, DATA_TYPE_IMAGETTE, info.cmp_mode_used == CMP_MODE_RAW,
465
4
              cmp_bit_to_byte(info.cmp_size));
466
4
      if (!ent_size)
467
0
        goto fail;
468
4
      decomp_entity = calloc(1, ent_size);
469
4
      if (!decomp_entity) {
470
0
        fprintf(stderr, "%s: Error allocating memory for decompression input buffer.\n", PROGRAM_NAME);
471
0
        goto fail;
472
0
      }
473
4
      ent_size = cmp_ent_create(decomp_entity, DATA_TYPE_IMAGETTE, info.cmp_mode_used == CMP_MODE_RAW,
474
4
              cmp_bit_to_byte(info.cmp_size));
475
4
      if (!ent_size)
476
0
        goto fail;
477
478
4
      f_size = read_file8(data_file_name, cmp_ent_get_data_buf(decomp_entity),
479
4
              cmp_bit_to_byte(info.cmp_size), io_flags);
480
4
      if (f_size < 0)
481
1
        goto fail;
482
483
3
      error = cmp_ent_write_rdcu_cmp_pars(decomp_entity, &info, NULL);
484
3
      if (error)
485
2
        goto fail;
486
127
    } else { /* read in compressed data with header */
487
127
      ssize_t size;
488
127
      size_t buf_size;
489
490
127
      printf("Importing compressed data file %s ... ", data_file_name);
491
127
      size = read_file_cmp_entity(data_file_name, NULL, 0, io_flags);
492
127
      if (size < 0 || size > INT32_MAX)
493
77
        goto fail;
494
      /* to be save allocate at least the size of the cmp_entity struct */
495
50
      buf_size = (size_t)size;
496
50
      if (buf_size < sizeof(struct cmp_entity))
497
44
        buf_size = sizeof(struct cmp_entity);
498
499
50
      decomp_entity = calloc(1, buf_size);
500
50
      if (!decomp_entity) {
501
0
        fprintf(stderr, "%s: Error allocating memory for the compression entity buffer.\n", PROGRAM_NAME);
502
0
        goto fail;
503
0
      }
504
50
      size = read_file_cmp_entity(data_file_name, decomp_entity,
505
50
                (uint32_t)size, io_flags);
506
50
      if (size < 0)
507
29
        goto fail;
508
509
21
      if (io_flags & CMP_IO_VERBOSE_EXTRA) {
510
0
        cmp_ent_print(decomp_entity);
511
0
        printf("\n");
512
0
      }
513
514
21
    }
515
22
    if (cmp_ent_get_data_type(decomp_entity) == DATA_TYPE_CHUNK)
516
2
      cmp_type = CMP_TYPE_CHUNK;
517
20
    else
518
20
      cmp_type = CMP_TYPE_RDCU;
519
520
22
    printf("DONE\n");
521
22
  }
522
523
551
  if (model_file_name && !guess_operation &&
524
551
      ((cmp_operation && !model_mode_is_used(rcfg.cmp_mode)) ||
525
170
       (!cmp_operation && !model_mode_is_used(cmp_ent_get_cmp_mode(decomp_entity)))))
526
115
    printf("Warring: Model file (-m option) specified but no model is used.\n");
527
528
  /* read in model */
529
551
  if ((cmp_operation && model_mode_is_used(rcfg.cmp_mode)) ||
530
551
      (!cmp_operation && model_mode_is_used(cmp_ent_get_cmp_mode(decomp_entity))) ||
531
551
      (guess_operation && model_file_name)) {
532
79
    ssize_t size;
533
79
    uint32_t model_size;
534
535
79
    printf("Importing model file %s ... ", model_file_name ? model_file_name : "");
536
79
    if (!model_file_name) {
537
2
      fprintf(stderr, "%s: No model file (-m option) specified.\n", PROGRAM_NAME);
538
2
      goto fail;
539
2
    }
540
541
77
    if (cmp_operation || guess_operation)
542
71
      model_size = input_size;
543
6
    else
544
6
      model_size = cmp_ent_get_original_size(decomp_entity);
545
546
77
    size = read_file_data(model_file_name, cmp_type, NULL,
547
77
              model_size, io_flags);
548
77
    if (size < 0)
549
8
      goto fail;
550
69
    if (size < (ssize_t)model_size) {
551
1
      fprintf(stderr, "%s: %s: Error: The files do not contain enough data. Expected: 0x%x, has 0x%x.\n",
552
1
        PROGRAM_NAME, model_file_name, model_size, (uint32_t)size);
553
1
      goto fail;
554
1
    }
555
68
    if (size != (ssize_t)model_size) {
556
1
      fprintf(stderr, "%s: %s: Error: Model file size does not match original data size.\n", PROGRAM_NAME, model_file_name);
557
1
      goto fail;
558
1
    }
559
560
67
    input_model_buf = malloc(model_size);
561
67
    if (!input_model_buf) {
562
0
      fprintf(stderr, "%s: Error allocating memory for model buffer.\n", PROGRAM_NAME);
563
0
      goto fail;
564
0
    }
565
566
67
    size = read_file_data(model_file_name, cmp_type, input_model_buf,
567
67
              model_size, io_flags);
568
67
    if (size < 0)
569
1
      goto fail;
570
571
572
66
    printf("DONE\n");
573
574
66
    rcfg.model_buf = input_model_buf;
575
66
    rcfg.icu_new_model_buf = input_model_buf; /* in-place model update */
576
66
  }
577
578
538
  if (guess_operation) {
579
56
    error = guess_cmp_pars(&rcfg, &chunk_par, input_size,
580
56
               guess_option, guess_level_str);
581
482
  } else if (cmp_operation) {
582
463
    if (cmp_type == CMP_TYPE_CHUNK)
583
285
      error = compression_of_chunk(rcfg.input_buf, input_size,
584
285
                 input_model_buf, &chunk_par);
585
178
    else
586
178
      error = compression_for_rdcu(&rcfg);
587
463
  } else {
588
19
    error = decompression(decomp_entity, input_model_buf);
589
19
  }
590
538
  if (error)
591
273
    goto fail;
592
593
  /* write our the updated model for compressed or decompression */
594
265
  if (!guess_operation &&
595
265
      ((cmp_operation && model_mode_is_used(rcfg.cmp_mode)) ||
596
219
      (!cmp_operation && model_mode_is_used(cmp_ent_get_cmp_mode(decomp_entity))))) {
597
18
    uint32_t model_size;
598
599
18
    printf("Write updated model to file %s_upmodel.dat ... ", output_prefix);
600
18
    if (cmp_operation)
601
16
      model_size = input_size;
602
2
    else
603
2
      model_size = cmp_ent_get_original_size(decomp_entity);
604
605
606
18
    error = write_input_data_to_file(input_model_buf, model_size, cmp_type,
607
18
             output_prefix, "_upmodel.dat", io_flags);
608
18
    if (error)
609
0
      goto fail;
610
18
    printf("DONE\n");
611
18
  }
612
613
265
  free(rcfg.input_buf);
614
265
  free(decomp_entity);
615
265
  free(input_model_buf);
616
617
265
  return EXIT_SUCCESS;
618
619
1.17k
fail:
620
1.17k
  printf("FAILED\n");
621
622
1.17k
  free(rcfg.input_buf);
623
1.17k
  free(decomp_entity);
624
1.17k
  free(input_model_buf);
625
626
1.17k
  return EXIT_FAILURE;
627
265
}
628
629
630
/**
631
 * @brief find a good set of compression parameters for a given dataset
632
 */
633
634
static int guess_cmp_pars(struct rdcu_cfg *rcfg, struct cmp_par *chunk_par,
635
        uint32_t input_size, const char *guess_option,
636
        const char *guess_level_str)
637
56
{
638
56
  int error;
639
56
  uint32_t cmp_size_bit;
640
56
  double cr MAYBE_UNUSED;
641
56
  enum cmp_data_type data_type;
642
56
  char *endptr;
643
56
  int guess_level;
644
645
56
  if (guess_level_str) {
646
0
    long number = strtol(guess_level_str, &endptr, 10);
647
648
0
    if (errno != 0 || *endptr != '\0' || number < INT_MIN || number > INT_MAX) {
649
0
      printf("Invalid guess level number: %s\n", guess_level_str);
650
0
      return -1;
651
0
    }
652
0
    guess_level = (int)number;
653
56
  } else {
654
56
    guess_level = DEFAULT_GUESS_LEVEL;
655
56
  }
656
657
56
  printf("Search for a good set of compression parameters (level: %d) ... ", guess_level);
658
56
  fflush(stdout);
659
56
  if (guess_option && !case_insensitive_compare(guess_option, "rdcu")) {
660
10
    if (add_rdcu_pars)
661
0
      data_type = DATA_TYPE_IMAGETTE_ADAPTIVE;
662
10
    else
663
10
      data_type = DATA_TYPE_IMAGETTE;
664
10
    if (rcfg->model_buf)
665
9
      rcfg->cmp_mode = CMP_GUESS_DEF_MODE_MODEL;
666
1
    else
667
1
      rcfg->cmp_mode = CMP_GUESS_DEF_MODE_DIFF;
668
46
  } else if (guess_option && !case_insensitive_compare(guess_option, "chunk")) {
669
0
    data_type = DATA_TYPE_CHUNK;
670
46
  } else {
671
46
    data_type = DATA_TYPE_IMAGETTE;
672
46
    error = cmp_mode_parse(guess_option, &rcfg->cmp_mode);
673
46
    if (error) {
674
6
      fprintf(stderr, "%s: Error: unknown guess option: %s\n", PROGRAM_NAME, guess_option);
675
6
      return -1;
676
6
    }
677
46
  }
678
50
  if (model_mode_is_used(rcfg->cmp_mode) && !rcfg->model_buf) {
679
4
    fprintf(stderr, "%s: Error: model mode needs model data (-m option)\n", PROGRAM_NAME);
680
4
    return -1;
681
4
  }
682
683
46
  if (data_type == DATA_TYPE_CHUNK) {
684
0
    uint32_t result = cmp_guess_chunk(rcfg->input_buf, input_size,
685
0
          rcfg->model_buf, chunk_par, guess_level);
686
687
0
    if (cmp_is_error(result))
688
0
      return -1;
689
690
0
    cmp_size_bit = 8 * result;
691
0
    printf("DONE\n");
692
693
0
    printf("Write the guessed compression chunk parameters to file %s.par ... ", output_prefix);
694
0
    error = cmp_par_fo_file(chunk_par, output_prefix, io_flags & CMP_IO_VERBOSE);
695
0
    if (error)
696
0
      return -1;
697
46
  } else {
698
46
    input_size = rcfg->samples * sizeof(uint16_t);
699
46
    cmp_size_bit = cmp_guess(rcfg, guess_level);
700
46
    if (!cmp_size_bit)
701
0
      return -1;
702
46
    if (include_cmp_header)
703
41
      cmp_size_bit = CHAR_BIT * (cmp_bit_to_byte(cmp_size_bit) +
704
41
        cmp_ent_cal_hdr_size(data_type, rcfg->cmp_mode == CMP_MODE_RAW));
705
46
    printf("DONE\n");
706
707
46
    printf("Write the guessed compression configuration to file %s.cfg ... ", output_prefix);
708
46
    error = cmp_cfg_fo_file(rcfg, output_prefix, io_flags & CMP_IO_VERBOSE, add_rdcu_pars);
709
46
    if (error)
710
0
      return -1;
711
46
  }
712
713
46
  printf("DONE\n");
714
715
46
  cr = (8.0 * input_size)/cmp_size_bit;
716
46
  printf("Guessed parameters can compress the data with a CR of %.2f.\n", cr);
717
718
46
  return 0;
719
46
}
720
721
722
/**
723
 * @brief generate packets to setup an RDCU compression
724
 */
725
726
static int gen_rdcu_write_pkts(const struct rdcu_cfg *rcfg)
727
0
{
728
0
  int error;
729
730
0
  error = init_rmap_pkt_to_file();
731
0
  if (error) {
732
0
    fprintf(stderr, "%s: Read RMAP packet config file .rdcu_pkt_mode_cfg failed.\n",
733
0
      PROGRAM_NAME);
734
0
    return -1;
735
0
  }
736
737
0
  if (last_info_file_name) {
738
    /* generation of packets for parallel read/write RDCU setup */
739
0
    struct cmp_info last_info = {0};
740
741
0
    error  = cmp_info_read(last_info_file_name, &last_info, io_flags & CMP_IO_VERBOSE);
742
0
    if (error) {
743
0
      fprintf(stderr, "%s: %s: Importing last decompression information file failed.\n",
744
0
        PROGRAM_NAME, last_info_file_name);
745
0
      return -1;
746
0
    }
747
748
0
    error = gen_rdcu_parallel_pkts(rcfg, &last_info);
749
0
    if (error)
750
0
      return -1;
751
0
  }
752
753
  /* generation of packets for non-parallel read/write RDCU setup */
754
0
  error = gen_write_rdcu_pkts(rcfg);
755
0
  if (error)
756
0
    return -1;
757
758
0
  return 0;
759
0
}
760
761
762
/**
763
 * return a current PLATO timestamp
764
 */
765
766
static uint64_t return_timestamp(void)
767
423
{
768
423
  return cmp_ent_create_timestamp(NULL);
769
423
}
770
771
772
/**
773
 * @brief compress chunk data and write the results to files
774
 */
775
776
static int compression_of_chunk(const void *chunk, uint32_t size, void *model,
777
        const struct cmp_par *chunk_par)
778
285
{
779
285
  uint32_t bound = compress_chunk_cmp_size_bound(chunk, size);
780
285
  uint32_t *cmp_data;
781
285
  uint32_t cmp_size;
782
285
  int error = 0;
783
784
285
  compress_chunk_init(&return_timestamp, cmp_tool_gen_version_id(CMP_TOOL_VERSION));
785
786
285
  if (!bound)
787
0
    return -1;
788
285
  cmp_data = calloc(1, bound);
789
285
  if (cmp_data == NULL) {
790
0
    fprintf(stderr, "%s: Error allocating memory for output buffer.\n", PROGRAM_NAME);
791
0
    return -1;
792
0
  }
793
794
285
  printf("Compress chunk data ... ");
795
285
  cmp_size = compress_chunk(chunk, size, model, model,
796
285
          cmp_data, bound, chunk_par);
797
285
  if (cmp_is_error(cmp_size))
798
147
    goto cmp_chunk_fail;
799
800
138
  cmp_size = compress_chunk_set_model_id_and_counter(cmp_data, cmp_size,
801
138
      (uint16_t)model_id, (uint8_t)model_counter);
802
138
  if (cmp_is_error(cmp_size))
803
0
    goto cmp_chunk_fail;
804
805
138
  printf("DONE\nWrite compressed data to file %s.cmp ... ", output_prefix);
806
138
  error = write_data_to_file(cmp_data, cmp_size, output_prefix, ".cmp", io_flags);
807
808
285
cmp_chunk_fail:
809
285
  free(cmp_data);
810
285
  cmp_data = NULL;
811
285
  if (cmp_is_error(cmp_size)) {
812
147
    fprintf(stderr, "%s: %s.\n", PROGRAM_NAME, cmp_get_error_name(cmp_size));
813
147
    printf("FAILED\n");
814
147
    return (int)cmp_get_error_code(cmp_size);
815
147
  }
816
138
  if (error) {
817
0
    printf("FAILED\n");
818
0
    return -1;
819
0
  }
820
138
  printf("DONE\n");
821
138
  return 0;
822
138
}
823
824
825
/**
826
 * @brief compress the data and write the results to files
827
 */
828
829
static int compression_for_rdcu(struct rdcu_cfg *rcfg)
830
178
{
831
178
  uint64_t start_time = cmp_ent_create_timestamp(NULL);
832
178
  enum cmp_data_type data_type = add_rdcu_pars ?
833
178
    DATA_TYPE_IMAGETTE_ADAPTIVE : DATA_TYPE_IMAGETTE;
834
178
  uint32_t cmp_size;
835
178
  int error;
836
178
  uint32_t cmp_size_byte, out_buf_size;
837
178
  size_t s;
838
178
  struct cmp_entity *cmp_entity = NULL;
839
178
  void *data_to_write_to_file;
840
178
  struct cmp_info info;
841
842
178
  if (rcfg->buffer_length == 0) {
843
149
    rcfg->buffer_length = (rcfg->samples+1) * BUFFER_LENGTH_DEF_FAKTOR; /* +1 to prevent malloc(0)*/
844
149
    printf("No buffer_length parameter set. Use buffer_length = %u as compression buffer size.\n",
845
149
           rcfg->buffer_length);
846
149
  }
847
848
178
  if (rdcu_pkt_mode) {
849
0
    void *tmp = rcfg->icu_new_model_buf;
850
851
0
    rcfg->icu_new_model_buf = NULL;
852
0
    printf("Generate compression setup packets ...\n");
853
0
    error = gen_rdcu_write_pkts(rcfg);
854
0
    if (error)
855
0
      goto error_cleanup;
856
0
    printf("... DONE\n");
857
0
    rcfg->icu_new_model_buf = tmp;
858
0
  }
859
860
178
  printf("Compress data ... ");
861
862
178
  out_buf_size = rcfg->buffer_length * sizeof(uint16_t);
863
178
  if (out_buf_size > CMP_ENTITY_MAX_SIZE * BUFFER_LENGTH_DEF_FAKTOR) {
864
1
    fprintf(stderr, "%s: Error buffer_length parameter to large.\n", PROGRAM_NAME);
865
1
    goto error_cleanup;
866
1
  }
867
868
177
  cmp_entity = calloc(1, out_buf_size + sizeof(struct cmp_entity));
869
177
  if (cmp_entity == NULL) {
870
0
    fprintf(stderr, "%s: Error allocating memory for output buffer.\n", PROGRAM_NAME);
871
0
    goto error_cleanup;
872
0
  }
873
177
  s = cmp_ent_create(cmp_entity, data_type, rcfg->cmp_mode == CMP_MODE_RAW, out_buf_size);
874
177
  if (!s) {
875
2
    fprintf(stderr, "%s: error occurred while creating the compression entity header.\n", PROGRAM_NAME);
876
2
    goto error_cleanup;
877
2
  }
878
175
  rcfg->icu_output_buf = cmp_ent_get_data_buf(cmp_entity);
879
880
175
  cmp_size = compress_like_rdcu(rcfg, &info);
881
175
  if (cmp_is_error(cmp_size)) {
882
59
    if (cmp_get_error_code(cmp_size) == CMP_ERROR_SMALL_BUFFER)
883
22
      fprintf(stderr, "Error: The buffer for the compressed data is too small to hold the compressed data. Try a larger buffer_length parameter.\n");
884
59
    goto error_cleanup;
885
59
  }
886
887
116
  if (!model_counter && model_mode_is_used(rcfg->cmp_mode))
888
13
    model_counter++;
889
890
116
  s = cmp_ent_create(cmp_entity, data_type, rcfg->cmp_mode == CMP_MODE_RAW, cmp_bit_to_byte(cmp_size));
891
116
  if (!s) {
892
0
    fprintf(stderr, "%s: error occurred while creating the compression entity header.\n", PROGRAM_NAME);
893
0
    goto error_cleanup;
894
0
  }
895
116
  error = cmp_ent_set_version_id(cmp_entity, cmp_tool_gen_version_id(CMP_TOOL_VERSION));
896
116
  error |= cmp_ent_set_start_timestamp(cmp_entity, start_time);
897
116
  error |= cmp_ent_set_end_timestamp(cmp_entity, cmp_ent_create_timestamp(NULL));
898
116
  error |= cmp_ent_set_model_id(cmp_entity, model_id);
899
116
  error |= cmp_ent_set_model_counter(cmp_entity, model_counter);
900
116
  error |= cmp_ent_write_rdcu_cmp_pars(cmp_entity, &info, rcfg);
901
116
  if (error) {
902
38
    fprintf(stderr, "%s: error occurred while creating the compression entity header.\n", PROGRAM_NAME);
903
38
    goto error_cleanup;
904
38
  }
905
906
78
  if (include_cmp_header) {
907
23
    data_to_write_to_file = cmp_entity;
908
23
    cmp_size_byte = cmp_ent_get_size(cmp_entity);
909
55
  } else {
910
55
    data_to_write_to_file = cmp_ent_get_data_buf(cmp_entity);
911
55
    cmp_size_byte = cmp_ent_get_cmp_data_size(cmp_entity);
912
55
  }
913
914
78
  printf("DONE\n");
915
916
78
  if (rdcu_pkt_mode) {
917
0
    printf("Generate the read results packets ... ");
918
0
    error = gen_read_rdcu_pkts(&info);
919
0
    if (error)
920
0
      goto error_cleanup;
921
0
    printf("DONE\n");
922
0
  }
923
924
78
  printf("Write compressed data to file %s.cmp ... ", output_prefix);
925
78
  error = write_data_to_file(data_to_write_to_file, cmp_size_byte,
926
78
           output_prefix, ".cmp", io_flags);
927
78
  if (error)
928
0
    goto error_cleanup;
929
78
  printf("DONE\n");
930
931
78
  if (!include_cmp_header) {
932
55
    printf("Write decompression information to file %s.info ... ",
933
55
           output_prefix);
934
55
    error = cmp_info_to_file(&info, output_prefix, add_rdcu_pars);
935
55
    if (error)
936
0
      goto error_cleanup;
937
55
    printf("DONE\n");
938
939
55
    if (io_flags & CMP_IO_VERBOSE) {
940
0
      printf("\n");
941
0
      print_cmp_info(&info);
942
0
      printf("\n");
943
0
    }
944
55
  }
945
946
78
  free(cmp_entity);
947
78
  rcfg->icu_output_buf = NULL;
948
949
78
  return 0;
950
951
100
error_cleanup:
952
100
  free(cmp_entity);
953
100
  rcfg->icu_output_buf = NULL;
954
955
100
  return -1;
956
78
}
957
958
959
/**
960
 * @brief decompress the data and write the results in file(s)
961
 */
962
963
static int decompression(const struct cmp_entity *ent, uint16_t *input_model_buf)
964
19
{
965
19
  int error;
966
19
  int decomp_size;
967
19
  uint16_t *decomp_output;
968
19
  enum cmp_type cmp_type;
969
970
19
  printf("Decompress data ... ");
971
972
19
  decomp_size = decompress_cmp_entiy(ent, input_model_buf, input_model_buf, NULL);
973
19
  if (decomp_size < 0)
974
16
    return -1;
975
3
  if (decomp_size == 0) {
976
3
    printf("\nWarring: No data are decompressed.\n... ");
977
3
    printf("DONE\n");
978
3
    return 0;
979
3
  }
980
981
0
  decomp_output = malloc((size_t)decomp_size);
982
0
  if (decomp_output == NULL) {
983
0
    fprintf(stderr, "%s: Error allocating memory for decompressed data.\n", PROGRAM_NAME);
984
0
    return -1;
985
0
  }
986
987
0
  decomp_size = decompress_cmp_entiy(ent, input_model_buf, input_model_buf, decomp_output);
988
0
  if (decomp_size <= 0) {
989
0
    free(decomp_output);
990
0
    return -1;
991
0
  }
992
993
0
  printf("DONE\n");
994
995
0
  printf("Write decompressed data to file %s.dat ... ", output_prefix);
996
997
0
  if (cmp_ent_get_data_type(ent) == DATA_TYPE_CHUNK)
998
0
    cmp_type = CMP_TYPE_CHUNK;
999
0
  else
1000
0
    cmp_type = CMP_TYPE_RDCU;
1001
0
  error = write_input_data_to_file(decomp_output, (uint32_t)decomp_size, cmp_type,
1002
0
           output_prefix, ".dat", io_flags);
1003
1004
0
  free(decomp_output);
1005
0
  if (error)
1006
0
    return -1;
1007
1008
0
  printf("DONE\n");
1009
1010
0
  return 0;
1011
0
}
1012
1013
1014
/**
1015
 * @brief create a default configuration for a compression data type
1016
 */
1017
1018
static void cmp_cfg_create_default(struct rdcu_cfg *rcfg, enum cfg_default_opt mode)
1019
18
{
1020
18
  if (!rcfg) /* nothing to do */
1021
0
    return;
1022
1023
18
  switch (mode) {
1024
16
  case MODEL_CFG:
1025
16
    rdcu_cfg_create(rcfg, CMP_DEF_IMA_MODEL_CMP_MODE, CMP_DEF_IMA_MODEL_MODEL_VALUE,
1026
16
        CMP_DEF_IMA_MODEL_LOSSY_PAR);
1027
16
    rdcu_cfg_buffers(rcfg, NULL, 0, NULL, CMP_DEF_IMA_MODEL_RDCU_DATA_ADR,
1028
16
        CMP_DEF_IMA_MODEL_RDCU_MODEL_ADR, CMP_DEF_IMA_MODEL_RDCU_UP_MODEL_ADR,
1029
16
        CMP_DEF_IMA_MODEL_RDCU_BUFFER_ADR, 0);
1030
16
    rdcu_cfg_imagette_default(rcfg);
1031
16
    break;
1032
2
  case DIFF_CFG:
1033
2
    rdcu_cfg_create(rcfg, CMP_DEF_IMA_DIFF_CMP_MODE,
1034
2
        CMP_DEF_IMA_DIFF_MODEL_VALUE,
1035
2
        CMP_DEF_IMA_DIFF_LOSSY_PAR);
1036
2
    rdcu_cfg_buffers(rcfg, NULL, 0, NULL, CMP_DEF_IMA_DIFF_RDCU_DATA_ADR,
1037
2
        CMP_DEF_IMA_DIFF_RDCU_MODEL_ADR, CMP_DEF_IMA_DIFF_RDCU_UP_MODEL_ADR,
1038
2
        CMP_DEF_IMA_DIFF_RDCU_BUFFER_ADR, 0);
1039
2
    rdcu_cfg_imagette_default(rcfg);
1040
2
    break;
1041
18
  }
1042
18
}