Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
cmp_tool
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Package registry
Model registry
Operate
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Dominik Loidolt
cmp_tool
Commits
410511e0
Commit
410511e0
authored
Mar 11, 2024
by
Dominik Loidolt
Browse files
Options
Downloads
Patches
Plain Diff
Add chunk compression to cmp_tool
parent
9d9d8ecf
No related branches found
No related tags found
1 merge request
!27
Add chunk compression to cmp_tool
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
meson.build
+1
-1
1 addition, 1 deletion
meson.build
programs/cmp_io.c
+70
-106
70 additions, 106 deletions
programs/cmp_io.c
programs/cmp_io.h
+2
-1
2 additions, 1 deletion
programs/cmp_io.h
programs/cmp_tool.c
+78
-14
78 additions, 14 deletions
programs/cmp_tool.c
with
151 additions
and
122 deletions
meson.build
+
1
−
1
View file @
410511e0
project
(
'cmp_tool'
,
'c'
,
version
:
'0.12-b
2
'
,
version
:
'0.12-b
3
'
,
meson_version
:
'>= 0.56'
,
license
:
'GPL-2.0'
,
default_options
:
[
...
...
This diff is collapsed.
Click to expand it.
programs/cmp_io.c
+
70
−
106
View file @
410511e0
...
...
@@ -31,6 +31,7 @@
#include
"cmp_io.h"
#include
<cmp_support.h>
#include
<cmp_chunk.h>
#include
<rdcu_cmd.h>
#include
<byteorder.h>
#include
<cmp_data_types.h>
...
...
@@ -65,6 +66,7 @@ static const struct {
{
DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE
,
"DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE"
},
{
DATA_TYPE_F_CAM_OFFSET
,
"DATA_TYPE_F_CAM_OFFSET"
},
{
DATA_TYPE_F_CAM_BACKGROUND
,
"DATA_TYPE_F_CAM_BACKGROUND"
},
{
DATA_TYPE_CHUNK
,
"DATA_TYPE_CHUNK"
},
{
DATA_TYPE_UNKNOWN
,
"DATA_TYPE_UNKNOWN"
}
};
...
...
@@ -528,22 +530,31 @@ int cmp_mode_parse(const char *cmp_mode_str, enum cmp_mode *cmp_mode)
* @param fp FILE pointer
* @param cfg compression configuration structure holding the read in
* configuration
* @param par chunk compression parameters structure holding the read in
* chunk configuration
*
* @returns 0 on success, error otherwise
*/
static
int
parse_cfg
(
FILE
*
fp
,
struct
cmp_cfg
*
cfg
)
static
int
parse_cfg
(
FILE
*
fp
,
struct
cmp_cfg
*
cfg
,
struct
cmp_par
*
par
)
{
#define chunk_parse_uint32_parmter(parmter) \
if (!strcmp(token1, #parmter)) { \
if (atoui32(token1, token2, &par->parmter)) \
return -1; \
cfg->data_type = DATA_TYPE_CHUNK; \
continue; \
}
char
*
token1
,
*
token2
;
char
line
[
MAX_CONFIG_LINE
];
enum
{
CMP_MODE
,
SAMPLES
,
BUFFER_LENGTH
,
LAST_ITEM
};
int
j
,
must_read_items
[
LAST_ITEM
]
=
{
0
};
if
(
!
fp
)
return
-
1
;
abort
()
;
if
(
!
cfg
)
return
-
1
;
abort
()
;
while
(
fgets
(
line
,
sizeof
(
line
),
fp
)
!=
NULL
)
{
if
(
line
[
0
]
==
'#'
||
line
[
0
]
==
'\n'
)
...
...
@@ -576,6 +587,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
must_read_items
[
CMP_MODE
]
=
1
;
if
(
cmp_mode_parse
(
token2
,
&
cfg
->
cmp_mode
))
return
-
1
;
par
->
cmp_mode
=
cfg
->
cmp_mode
;
continue
;
}
if
(
!
strcmp
(
token1
,
"golomb_par"
))
{
...
...
@@ -591,6 +603,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
if
(
!
strcmp
(
token1
,
"model_value"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
model_value
))
return
-
1
;
par
->
model_value
=
cfg
->
model_value
;
continue
;
}
if
(
!
strcmp
(
token1
,
"round"
))
{
...
...
@@ -618,98 +631,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"cmp_par_exp_flags"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
cmp_par_exp_flags
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"spill_exp_flags"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
spill_exp_flags
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"cmp_par_fx"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
cmp_par_fx
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"spill_fx"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
spill_fx
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"cmp_par_ncob"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
cmp_par_ncob
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"spill_ncob"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
spill_ncob
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"cmp_par_efx"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
cmp_par_efx
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"spill_efx"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
spill_efx
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"cmp_par_ecob"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
cmp_par_ecob
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"spill_ecob"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
spill_ecob
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"cmp_par_fx_cob_variance"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
cmp_par_fx_cob_variance
))
return
-
1
;
continue
;
}
if
(
!
strcmp
(
token1
,
"spill_fx_cob_variance"
))
{
if
(
atoui32
(
token1
,
token2
,
&
cfg
->
spill_fx_cob_variance
))
return
-
1
;
continue
;
}
#if 0
if (!strcmp(token1, "cmp_par_mean")) {
if (atoui32(token1, token2, &cfg->cmp_par_mean))
return -1;
continue;
}
if (!strcmp(token1, "spill_mean")) {
if (atoui32(token1, token2, &cfg->spill_mean))
return -1;
continue;
}
if (!strcmp(token1, "cmp_par_variance")) {
if (atoui32(token1, token2, &cfg->cmp_par_variance))
return -1;
continue;
}
if (!strcmp(token1, "spill_variance")) {
if (atoui32(token1, token2, &cfg->spill_variance))
return -1;
continue;
}
if (!strcmp(token1, "cmp_par_pixels_error")) {
if (atoui32(token1, token2, &cfg->cmp_par_pixels_error))
return -1;
continue;
}
if (!strcmp(token1, "spill_pixels_error")) {
if (atoui32(token1, token2, &cfg->spill_pixels_error))
return -1;
continue;
}
#endif
if
(
!
strcmp
(
token1
,
"rdcu_data_adr"
))
{
int
i
=
sram_addr_to_int
(
token2
);
...
...
@@ -766,8 +688,46 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
must_read_items
[
BUFFER_LENGTH
]
=
1
;
continue
;
}
/* chunk_parse_uint32_parmter(model_value); */
chunk_parse_uint32_parmter
(
lossy_par
);
chunk_parse_uint32_parmter
(
nc_imagette
);
chunk_parse_uint32_parmter
(
s_exp_flags
);
chunk_parse_uint32_parmter
(
s_fx
);
chunk_parse_uint32_parmter
(
s_ncob
);
chunk_parse_uint32_parmter
(
s_efx
);
chunk_parse_uint32_parmter
(
s_ecob
);
chunk_parse_uint32_parmter
(
l_exp_flags
);
chunk_parse_uint32_parmter
(
l_fx
);
chunk_parse_uint32_parmter
(
l_ncob
);
chunk_parse_uint32_parmter
(
l_efx
);
chunk_parse_uint32_parmter
(
l_ecob
);
chunk_parse_uint32_parmter
(
l_fx_cob_variance
);
chunk_parse_uint32_parmter
(
saturated_imagette
);
chunk_parse_uint32_parmter
(
nc_offset_mean
);
chunk_parse_uint32_parmter
(
nc_offset_variance
);
chunk_parse_uint32_parmter
(
nc_background_mean
);
chunk_parse_uint32_parmter
(
nc_background_variance
);
chunk_parse_uint32_parmter
(
nc_background_outlier_pixels
);
chunk_parse_uint32_parmter
(
smearing_mean
);
chunk_parse_uint32_parmter
(
smearing_variance_mean
);
chunk_parse_uint32_parmter
(
smearing_outlier_pixels
);
chunk_parse_uint32_parmter
(
fc_imagette
);
chunk_parse_uint32_parmter
(
fc_offset_mean
);
chunk_parse_uint32_parmter
(
fc_offset_variance
);
chunk_parse_uint32_parmter
(
fc_background_mean
);
chunk_parse_uint32_parmter
(
fc_background_variance
);
chunk_parse_uint32_parmter
(
fc_background_outlier_pixels
);
}
if
(
cfg
->
data_type
!=
DATA_TYPE_CHUNK
)
{
if
(
raw_mode_is_used
(
cfg
->
cmp_mode
))
if
(
must_read_items
[
CMP_MODE
]
==
1
&&
must_read_items
[
BUFFER_LENGTH
]
==
1
)
...
...
@@ -780,6 +740,7 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
return
-
1
;
}
}
}
return
0
;
}
...
...
@@ -791,12 +752,15 @@ static int parse_cfg(FILE *fp, struct cmp_cfg *cfg)
* @param file_name file containing the compression configuration file
* @param cfg compression configuration structure holding the read in
* configuration
* @param par chunk compression parameters structure holding the read
* in chunk configuration
* @param verbose_en print verbose output if not zero
*
* @returns 0 on success, error otherwise
*/
int
cmp_cfg_read
(
const
char
*
file_name
,
struct
cmp_cfg
*
cfg
,
int
verbose_en
)
int
cmp_cfg_read
(
const
char
*
file_name
,
struct
cmp_cfg
*
cfg
,
struct
cmp_par
*
par
,
int
verbose_en
)
{
int
err
;
FILE
*
fp
;
...
...
@@ -820,7 +784,7 @@ int cmp_cfg_read(const char *file_name, struct cmp_cfg *cfg, int verbose_en)
return
-
1
;
}
err
=
parse_cfg
(
fp
,
cfg
);
err
=
parse_cfg
(
fp
,
cfg
,
par
);
fclose
(
fp
);
if
(
err
)
return
err
;
...
...
This diff is collapsed.
Click to expand it.
programs/cmp_io.h
+
2
−
1
View file @
410511e0
...
...
@@ -24,6 +24,7 @@
#include
<string.h>
#include
<cmp_support.h>
#include
<cmp_chunk.h>
#include
<cmp_entity.h>
#define MAX_CONFIG_LINE 256
...
...
@@ -39,7 +40,7 @@
void
print_help
(
const
char
*
program_name
);
int
cmp_cfg_read
(
const
char
*
file_name
,
struct
cmp_cfg
*
cfg
,
int
verbose_en
);
int
cmp_cfg_read
(
const
char
*
file_name
,
struct
cmp_cfg
*
cfg
,
struct
cmp_par
*
par
,
int
verbose_en
);
int
cmp_info_read
(
const
char
*
file_name
,
struct
cmp_info
*
info
,
int
verbose_en
);
ssize_t
read_file8
(
const
char
*
file_name
,
uint8_t
*
buf
,
uint32_t
buf_size
,
int
flags
);
...
...
This diff is collapsed.
Click to expand it.
programs/cmp_tool.c
+
78
−
14
View file @
410511e0
...
...
@@ -27,6 +27,7 @@
#include
"cmp_tool-config.h"
#include
"cmp_io.h"
#include
"cmp_icu.h"
#include
"cmp_chunk.h"
#include
"cmp_rdcu.h"
#include
"decmp.h"
#include
"cmp_guess.h"
...
...
@@ -48,6 +49,9 @@ static enum cmp_data_type parse_data_type(const char *data_type_str);
static
int
guess_cmp_pars
(
struct
cmp_cfg
*
cfg
,
const
char
*
guess_cmp_mode
,
int
guess_level
);
/* compress chunk data and write the results to files */
static
int
compression_of_chunk
(
void
*
chunk
,
uint32_t
size
,
void
*
model
,
struct
cmp_par
*
chunk_par
);
/* compress the data and write the results to files */
static
int
compression
(
struct
cmp_cfg
*
cfg
,
struct
cmp_info
*
info
);
...
...
@@ -152,9 +156,12 @@ int main(int argc, char **argv)
struct
cmp_entity
*
decomp_entity
=
NULL
;
/* buffer containing the read in model */
uint16_t
*
input_model_buf
=
NULL
;
/* size of the data to be compressed and the model of it */
uint32_t
input_size
;
struct
cmp_info
info
=
{
0
};
/* decompression information struct */
struct
cmp_cfg
cfg
=
{
0
};
/* compressor configuration struct */
struct
cmp_par
chunk_par
=
{
0
};
/* compressor parameters for chunk compression */
cfg
.
data_type
=
DATA_TYPE_IMAGETTE
;
/* use imagette as default data type */
cfg
.
max_used_bits
=
&
MAX_USED_BITS_SAFE
;
/* define max_used_bits default */
...
...
@@ -327,12 +334,11 @@ int main(int argc, char **argv)
if
(
cmp_operation
||
guess_operation
)
{
ssize_t
size
;
uint32_t
input_size
;
if
(
cmp_operation
)
{
printf
(
"## Starting the compression ##
\n
"
);
printf
(
"Importing configuration file %s ... "
,
cfg_file_name
);
error
=
cmp_cfg_read
(
cfg_file_name
,
&
cfg
,
io_flags
&
CMP_IO_VERBOSE
);
error
=
cmp_cfg_read
(
cfg_file_name
,
&
cfg
,
&
chunk_par
,
io_flags
&
CMP_IO_VERBOSE
);
if
(
error
)
goto
fail
;
printf
(
"DONE
\n
"
);
...
...
@@ -342,6 +348,7 @@ int main(int argc, char **argv)
printf
(
"Importing data file %s ... "
,
data_file_name
);
/* count the samples in the data file when samples == 0 */
if
(
cfg
.
data_type
!=
DATA_TYPE_CHUNK
)
{
if
(
cfg
.
samples
==
0
)
{
int32_t
samples
;
...
...
@@ -356,6 +363,13 @@ int main(int argc, char **argv)
}
input_size
=
cmp_cal_size_of_data
(
cfg
.
samples
,
cfg
.
data_type
);
}
else
{
size
=
read_file_data
(
data_file_name
,
cfg
.
data_type
,
NULL
,
0
,
io_flags
);
if
(
size
<=
0
||
size
>
UINT32_MAX
)
/* empty file is treated as an error */
goto
fail
;
input_size
=
(
uint32_t
)
size
;
}
cfg
.
input_buf
=
malloc
(
input_size
);
if
(
!
cfg
.
input_buf
)
{
fprintf
(
stderr
,
"%s: Error allocating memory for input data buffer.
\n
"
,
PROGRAM_NAME
);
...
...
@@ -456,7 +470,7 @@ int main(int argc, char **argv)
if
(
cmp_operation
||
guess_operation
)
{
data_type
=
cfg
.
data_type
;
model_size
=
cmp_cal_size_of_data
(
cfg
.
samples
,
cfg
.
data_type
)
;
model_size
=
input_size
;
}
else
{
data_type
=
cmp_ent_get_data_type
(
decomp_entity
);
model_size
=
cmp_ent_get_original_size
(
decomp_entity
);
...
...
@@ -483,6 +497,10 @@ int main(int argc, char **argv)
if
(
error
)
goto
fail
;
}
else
if
(
cmp_operation
)
{
if
(
cfg
.
data_type
==
DATA_TYPE_CHUNK
)
error
=
compression_of_chunk
(
cfg
.
input_buf
,
input_size
,
input_model_buf
,
&
chunk_par
);
else
error
=
compression
(
&
cfg
,
&
info
);
if
(
error
)
goto
fail
;
...
...
@@ -502,7 +520,7 @@ int main(int argc, char **argv)
printf
(
"Write updated model to file %s_upmodel.dat ... "
,
output_prefix
);
if
(
cmp_operation
)
{
data_type
=
cfg
.
data_type
;
model_size
=
cmp_cal_size_of_data
(
cfg
.
samples
,
data_type
)
;
model_size
=
input_size
;
}
else
{
data_type
=
cmp_ent_get_data_type
(
decomp_entity
);
model_size
=
cmp_ent_get_original_size
(
decomp_entity
);
...
...
@@ -705,6 +723,52 @@ static int cmp_gernate_rdcu_info(const struct cmp_cfg *cfg, int cmp_size_bit,
}
/**
* @brief compress chunk data and write the results to files
*/
static
int
compression_of_chunk
(
void
*
chunk
,
uint32_t
size
,
void
*
model
,
struct
cmp_par
*
chunk_par
)
{
uint32_t
bound
=
compress_chunk_cmp_size_bound
(
chunk
,
size
);
uint32_t
*
cmp_data
;
int32_t
cmp_size
;
int
error
;
if
(
!
bound
)
return
-
1
;
cmp_data
=
calloc
(
1
,
bound
);
if
(
cmp_data
==
NULL
)
{
fprintf
(
stderr
,
"%s: Error allocating memory for output buffer.
\n
"
,
PROGRAM_NAME
);
return
-
1
;
}
printf
(
"Compress chunk data ... "
);
cmp_size
=
compress_chunk
(
chunk
,
size
,
model
,
model
,
cmp_data
,
bound
,
chunk_par
);
if
(
cmp_size
<
0
)
{
if
(
cmp_size
==
CMP_ERROR_SMALL_BUF
)
fprintf
(
stderr
,
"Error: The buffer for the compressed data is too small to hold the compressed data. Try a larger buffer_length parameter.
\n
"
);
free
(
cmp_data
);
cmp_data
=
NULL
;
printf
(
"FAILED
\n
"
);
return
-
1
;
}
printf
(
"DONE
\n
Write compressed data to file %s.cmp ... "
,
output_prefix
);
error
=
write_data_to_file
(
cmp_data
,
(
uint32_t
)
cmp_size
,
output_prefix
,
".cmp"
,
io_flags
);
free
(
cmp_data
);
cmp_data
=
NULL
;
if
(
error
)
{
printf
(
"FAILED
\n
"
);
return
-
1
;
}
printf
(
"DONE
\n
"
);
return
0
;
}
/**
* @brief compress the data and write the results to files
*/
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment