Restructuring of the reading of compressed data bitstream

Relaxation of the requirement that the data to be decompressed must be 4-byte aligned

lib/common/cmp_entity.c

@@ -1816,7 +1816,7 @@ int cmp_ent_write_cmp_pars(struct cmp_entity *ent, const struct cmp_cfg *cfg,
 	ent_cmp_data_size = cmp_ent_get_cmp_data_size(ent);
 
 	/* check if the entity can hold the compressed data */
-	if (ent_cmp_data_size < cmp_bit_to_4byte((unsigned int)cmp_size_bits)) {
+	if (ent_cmp_data_size < cmp_bit_to_byte((unsigned int)cmp_size_bits)) {
 		debug_print("Error: The entity size is to small to hold the compressed data.\n");
 		return -2;
 	}
@@ -1985,7 +1985,7 @@ int cmp_ent_write_rdcu_cmp_pars(struct cmp_entity *ent, const struct cmp_info *i
 
 	/* check if the entity can hold the compressed data */
 	ent_cmp_data_size = cmp_ent_get_cmp_data_size(ent);
-	if (ent_cmp_data_size < cmp_bit_to_4byte(info->cmp_size)) {
+	if (ent_cmp_data_size < cmp_bit_to_byte(info->cmp_size)) {
 		debug_print("Error: The entity size is to small to hold the compressed data.\n");
 		return -2;
 	}
@@ -2106,7 +2106,7 @@ uint32_t cmp_ent_build(struct cmp_entity *ent, uint32_t version_id,
 		       uint64_t start_time, uint64_t end_time, uint16_t model_id,
 		       uint8_t model_counter, const struct cmp_cfg *cfg, int cmp_size_bits)
 {
-	uint32_t cmp_size_bytes = cmp_bit_to_4byte((unsigned int)cmp_size_bits); /* TODO: do we need to round up to 4 bytes? */
+	uint32_t cmp_size_bytes = cmp_bit_to_byte((unsigned int)cmp_size_bits);
 	uint32_t hdr_size;
 
 	if (!cfg)

lib/common/cmp_support.c

@@ -366,12 +366,26 @@ uint32_t cmp_icu_max_spill(unsigned int cmp_par)
  * @param cmp_size_bit	compressed data size, measured in bits
  *
  * @returns the size in bytes to store the hole bitstream
+ */
+
+unsigned int cmp_bit_to_byte(unsigned int cmp_size_bit)
+{
+	return (cmp_size_bit + 7) / 8;
+}
+
+
+/**
+ * @brief calculate the need bytes to hold a bitstream
  * @note we round up the result to multiples of 4 bytes
+ *
+ * @param cmp_size_bit	compressed data size, measured in bits
+ *
+ * @returns the size in bytes to store the hole bitstream
  */
 
 unsigned int cmp_bit_to_4byte(unsigned int cmp_size_bit)
 {
-	return (((cmp_size_bit + 7) / 8) + 3) & ~0x3UL;
+	return (cmp_bit_to_byte(cmp_size_bit) + 3) & ~0x3UL;
 }
 
 

lib/common/cmp_support.h

@@ -269,6 +269,7 @@ struct fx_cob_par {
 int is_a_pow_of_2(unsigned int v);
 unsigned int ilog_2(uint32_t x);
 
+unsigned int cmp_bit_to_byte(unsigned int cmp_size_bit);
 unsigned int cmp_bit_to_4byte(unsigned int cmp_size_bit);
 
 int cmp_cfg_icu_is_invalid(const struct cmp_cfg *cfg);

lib/decompress/read_bitstream.h

+/**
+ * @file   read_bitstream.h
+ * @author Dominik Loidolt (dominik.loidolt@univie.ac.at)
+ * @date   2023
+ *
+ * @copyright GPLv2
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * @brief this library handles the reading from an MSB-first bitstream
+ *
+ * This API consists of small unitary functions, which must be inlined for best performance.
+ * Since link-time-optimization is not available for all compilers, these
+ * functions are defined into a .h to be included.
+ *
+ * Start by invoking bit_init_decoder(). A chunk of the bitstream is then stored
+ * into a local register. The local register size is 64 bits.  You can then retrieve
+ * bit-fields stored into the local register. The local register is explicitly
+ * reloaded from the memory with the bit_refill() function.
+ * A reload guarantees a minimum of 57 bits in the local register if the
+ * returned status is BIT_UNFINISHED.
+ * Otherwise, it can be less than that, so proceed accordingly.
+ * Checking if bit_decoder has reached its end can be performed with bit_end_of_stream().
+ *
+ * This is based on the bitstream part of the FiniteStateEntropy library, see:
+ * https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/lib/bitstream.h
+ * by @author Yann Collet
+ * As well as some ideas from this blog post:
+ * https://fgiesen.wordpress.com/2018/02/20/reading-bits-in-far-too-many-ways-part-2/
+ * by @author Fabian Giesen
+ */
+
 #ifndef READ_BITSTREAM_H
 #define READ_BITSTREAM_H
 
@@ -9,25 +47,12 @@
 #include "../common/byteorder.h"
 
 
-static __inline uint64_t bit_read_unaligned_64(const void* ptr)
-{
-	typedef __attribute__((aligned(1))) uint64_t unalign64;
-	return *(const unalign64*)ptr;
-}
-
-
-static __inline uint64_t bit_read_unalingned_be64(const void* ptr)
-{
-	return cpu_to_be64(bit_read_unaligned_64(ptr));
-}
-
 
 /**
- * @brief bitstream decoding context type
+ * @brief bitstream decoder context type
  */
 
-struct bit_decoder
-{
+struct bit_decoder {
 	uint64_t bit_container;
 	unsigned int bits_consumed;
 	const uint8_t *cursor;
@@ -35,42 +60,105 @@ struct bit_decoder
 };
 
 
+/**
+ * @brief bitstream decoder status, return type of bit_refill()
+ */
+
+enum bit_status {BIT_OVERFLOW, BIT_END_OF_BUFFER, BIT_ALL_READ_IN, BIT_UNFINISHED};
+
+
+/*
+ * bitstream decoder API
+ */
+
+static __inline size_t bit_init_decoder(struct bit_decoder *dec, const void *buf, size_t buf_size);
+static __inline uint64_t bit_peek_bits(const struct bit_decoder *dec, unsigned int nb_bits);
+static __inline void bit_consume_bits(struct bit_decoder *dec, unsigned int nb_bits);
+static __inline uint64_t bit_read_bits(struct bit_decoder *dec, unsigned int nb_bits);
+static __inline uint32_t bit_read_bits32(struct bit_decoder *dec, unsigned int nb_bits);
+static __inline uint32_t bit_read_bits32_sub_1(struct bit_decoder *dec, unsigned int nb_bits);
+static __inline unsigned int bit_end_of_stream(const struct bit_decoder *dec);
+static __inline int bit_refill(struct bit_decoder *dec);
+
+
+/*
+ * internal implementation
+ */
+
+static const uint32_t BIT_MASK[] = {
+	0,          1,          3,         7,         0xF,       0x1F,
+	0x3F,       0x7F,       0xFF,      0x1FF,     0x3FF,     0x7FF,
+	0xFFF,      0x1FFF,     0x3FFF,    0x7FFF,    0xFFFF,    0x1FFFF,
+	0x3FFFF,    0x7FFFF,    0xFFFFF,   0x1FFFFF,  0x3FFFFF,  0x7FFFFF,
+	0xFFFFFF,   0x1FFFFFF,  0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
+	0x3FFFFFFF, 0x7FFFFFFF, 0xFFFFFFFF}; /* up to 32 bits */
+#define BIT_MASK_SIZE ARRAY_SIZE(BIT_mask)
+
+
+/**
+ * @brief read 8 bytes of big-endian data from an unaligned address
+ *
+ * @param ptr pointer to the data (can be unaligned)
+ *
+ * @returns 64 bit data at mem_ptr address in big-endian byte order
+ */
+
+static __inline uint64_t bit_read_unaligned_64be(const void *ptr)
+{
+	typedef __attribute__((aligned(1))) uint64_t unalign64;
+	return cpu_to_be64(*(const unalign64*)ptr);
+}
+
+
+/**
+ * @brief initialize a bit_decoder
+ *
+ * @param dec		a pointer to an already allocated bit_decoder structure
+ * @param buf		start address of the bitstream buffer
+ * @param buf_size	size of the bitstream in bytes
+ *
+ * @returns size of stream (== src_size), or zero if a problem is detected
+ */
+
 static __inline size_t bit_init_decoder(struct bit_decoder *dec, const void *buf,
 					size_t buf_size)
 {
 	if (buf_size < 1) {
 		memset(dec, 0, sizeof(*dec));
+		dec->bits_consumed = sizeof(dec->bit_container)*8;
 		return 0;
 	}
 
 	dec->cursor = (const uint8_t *)buf;
 
-	if (buf_size >= sizeof(dec->bit_container)) {
+	if (buf_size >= sizeof(dec->bit_container)) {  /* normal case */
 		dec->bits_consumed = 0;
-		dec->bit_container = bit_read_unalingned_be64(dec->cursor);
+		dec->bit_container = bit_read_unaligned_64be(dec->cursor);
 		dec->limit_ptr = dec->cursor + buf_size - sizeof(dec->bit_container);
 	} else {
+		const uint8_t *ui8_p = (const uint8_t *)(buf);
+
 		dec->bits_consumed = (unsigned int)(sizeof(dec->bit_container) - buf_size) * 8;
 
-		dec->bit_container = (uint64_t)(((const uint8_t*)(buf))[0]) << 56;
+		dec->bit_container = (uint64_t)ui8_p[0] << 56;
 		switch (buf_size) {
 		case 7:
-			dec->bit_container += (uint64_t)(((const uint8_t*)(buf))[6]) <<  8;
+			dec->bit_container += (uint64_t)ui8_p[6] <<  8;
 			/* fall-through */
 		case 6:
-			dec->bit_container += (uint64_t)(((const uint8_t*)(buf))[5]) << 16;
+			dec->bit_container += (uint64_t)ui8_p[5] << 16;
 			/* fall-through */
 		case 5:
-			dec->bit_container += (uint64_t)(((const uint8_t*)(buf))[4]) << 24;
+			dec->bit_container += (uint64_t)ui8_p[4] << 24;
 			/* fall-through */
 		case 4:
-			dec->bit_container += (uint64_t)(((const uint8_t*)(buf))[3]) << 32;
+			dec->bit_container += (uint64_t)ui8_p[3] << 32;
 			/* fall-through */
 		case 3:
-			dec->bit_container += (uint64_t)(((const uint8_t*)(buf))[2]) << 40;
+			dec->bit_container += (uint64_t)ui8_p[2] << 40;
 			/* fall-through */
 		case 2:
-			dec->bit_container += (uint64_t)(((const uint8_t*)(buf))[1]) << 48;
+			dec->bit_container += (uint64_t)ui8_p[1] << 48;
 			/* fall-through */
 		default:
 			break;
@@ -79,18 +167,26 @@ static __inline size_t bit_init_decoder(struct bit_decoder *dec, const void* buf
 
 		dec->limit_ptr = dec->cursor;
 	}
-
 	return buf_size;
 }
 
 
+/**
+ * @brief provides next n bits from local register; local register is not modified
+ *
+ * @param dec		a pointer to a bit_decoder context
+ * @param nb_bits	number of bits to look; only works if 1 <= nb_bits <= 56
+ *
+ * @returns extracted value
+ */
+
 static __inline uint64_t bit_peek_bits(const struct bit_decoder *dec, unsigned int nb_bits)
 {
-	/* mask for the shift value register to prevent undefined behavior */
+	/* mask for the shift value register to prevent undefined behaviour */
 	uint32_t const reg_mask = 0x3F;
 
-	assert(nb_bits >= 1 && nb_bits <= (64 - 7)); /* TODO: why -7 */
-	assert(dec->bits_consumed + nb_bits <= 64);
+	assert(nb_bits >= 1 && nb_bits <= (64 - 7));
+	/* why -7: worst case refill can only put 56 bit in the bit_container */
 
 	/* shift out consumed bits; return the top nb_bits bits we want to peek */
 	return (dec->bit_container << (dec->bits_consumed & reg_mask)) >> (64-nb_bits);
@@ -99,33 +195,50 @@ static __inline uint64_t bit_peek_bits(const struct bit_decoder *dec, unsigned i
 
 /**
  * @brief count the leading ones in the local register; local register is not modified
- * @warning if all bits are consumed in local register (bitD->bitsConsumed  >= 64),
- *	the result is undefined
- * @param dec	a pointer to a bit_DStream_t context
- * @returns number of leading ones; up to maximum 63
+ *
+ * @param dec	pointer to a bit_decoder context
+ *
+ * @returns number of leading ones;
  */
 
-static __inline unsigned int bit_count_leading_ones(const struct bit_decoder* dec)
+static __inline unsigned int bit_peek_leading_ones(const struct bit_decoder *dec)
 {
-	/* mask for the shift value register to prevent undefined behavior */
+	/* mask for the shift value register to prevent undefined behaviour */
 	uint32_t const reg_mask = 0x3F;
 	/* shift out the bits we've already consumed */
-	uint64_t remaining_flip = ~(dec->bit_container << (dec->bits_consumed & reg_mask));
+	uint64_t const remaining_flip = ~(dec->bit_container << (dec->bits_consumed & reg_mask));
 
-	/* clzll(0) is undefined behavior */
-	if (remaining_flip)
-		return sizeof(dec->bit_container)*8;
-
-	return (unsigned int)__builtin_clzll(remaining_flip);
+	/* clzll(0) is undefined behaviour */
+	return remaining_flip ? (unsigned int)__builtin_clzll(remaining_flip) :
+		sizeof(dec->bit_container)*8;
 }
 
 
+/**
+ * @brief mark next n bits in the local register as consumed
+ *
+ * @param dec		pointer to a bit_decoder context
+ * @param nb_bits	number of bits to skip
+ */
+
 static __inline void bit_consume_bits(struct bit_decoder *dec, unsigned int nb_bits)
 {
 	dec->bits_consumed += nb_bits;
 }
 
 
+/**
+ * @brief read and consume next n bits from the local register
+ * @warning do not read more bits than the local register has unconsumed bits.
+ *	If you do this, the bit_refill function will return the BIT_OVERFLOW
+ *	status the next time the register is refilled.
+ *
+ * @param dec		pointer to a bit_decoder context
+ * @param nb_bits	number of bits to look; only works if 1 <= nb_bits <= 56
+ *
+ * @returns extracted value
+ */
+
 static __inline uint64_t bit_read_bits(struct bit_decoder *dec, unsigned int nb_bits)
 {
 	uint64_t const read_bits = bit_peek_bits(dec, nb_bits);
@@ -136,49 +249,120 @@ static __inline uint64_t bit_read_bits(struct bit_decoder *dec, unsigned int nb_
 
 
 /**
- * @brief Check if the end of the bitstream has been reached
- * @param dec	a bitstream decoding context
- * @returns 1 if DStream has _exactly_ reached its end (all bits consumed).
+ * @brief same as bit_read_bits32() but only returns 32 bit
+ * @warning do not read more bits than the local register has unconsumed bits.
+ *	If you do this, the bit_refill function will return the BIT_OVERFLOW
+ *	status the next time the register is refilled.
+ *
+ * @param dec		pointer to a bit_decoder context
+ * @param nb_bits	number of bits to read; only works if 1 <= nb_bits <= 32
+ *
+ * @returns extracted 32 bit value
  */
 
-static __inline unsigned int bit_end_of_stream(const struct bit_decoder* dec)
+static __inline uint32_t bit_read_bits32(struct bit_decoder *dec, unsigned int nb_bits)
 {
-    return ((dec->cursor == dec->limit_ptr) &&
-	    (dec->bits_consumed == sizeof(dec->bit_container)*8));
+	assert(nb_bits <= 32);
+
+	return (uint32_t)bit_read_bits(dec, nb_bits);
+}
+
+
+/**
+ * @brief same as bit_read_bits32() but subtract 1 from the extracted value
+ * @warning do not read more bits than the local register has unconsumed bits.
+ *	If you do this, the bit_refill function will return the BIT_OVERFLOW
+ *	status the next time the register is refilled.
+ *
+ * @param dec		pointer to a bit_decoder context
+ * @param nb_bits	number of bits to read; only works if nb_bits <= 32
+ *
+ * @returns extracted 32 bit value minus 1
+ *
+ * @note The difference to the bit_read_bits32() function with subtraction is
+ *	that the subtracted value is masked with nb_bits. E.g. if you read 4
+ *	bits from the bitstream and get 0 and then subtract 1, you get 0xFF
+ *	instead of 0xFFFFFFFF
+ */
+
+static __inline uint32_t bit_read_bits32_sub_1(struct bit_decoder *dec, unsigned int nb_bits)
+{
+	/* mask for the shift value register to prevent undefined behaviour */
+	uint32_t const reg_mask = sizeof(dec->bit_container)*8 - 1;
+	unsigned int const shift_bits = (64 - dec->bits_consumed - nb_bits) & reg_mask;
+	uint32_t bits_unmask;
+
+	assert(nb_bits <= 32);
+
+	bits_unmask = (uint32_t)(dec->bit_container >> shift_bits);
+	bit_consume_bits(dec, nb_bits);
+	return (bits_unmask - 1) & BIT_MASK[nb_bits];
 }
 
 
-enum {BIT_OVERFLOW, BIT_END_OF_BUFFER, BIT_ALL_READ_IN, BIT_UNFINISHED};
+/**
+ * @brief refill the local register from the buffer previously set in
+ *	bit_init_decoder()
+ *
+ * @param dec	a bitstream decoding context
+ *
+ * @note this function is safe, it guarantees that it does not read beyond
+ *	initialize buffer
+ *
+ * @returns the status of bit_decoder internal register;
+ *	BIT_UNFINISHED: internal register is filled with at least _57-bits_
+ *	BIT_END_OF_BUFFER: reached the end of the buffer, only some bits are left in the bitstream
+ *	BIT_ALL_READ_IN: _all_ bits of the buffer have been consumed
+ *	BIT_OVERFLOW: more bits have been consumed than contained in the local register
+ */
 
 static __inline int bit_refill(struct bit_decoder *dec)
 {
-	if (dec->bits_consumed > (sizeof(dec->bit_container)*8))
+	unsigned int const bytes_consumed = dec->bits_consumed >> 3;
+
+	if (dec->bits_consumed > sizeof(dec->bit_container)*8)
 		return BIT_OVERFLOW;
-	if (dec->cursor < dec->limit_ptr) {
+
+	if (dec->cursor + bytes_consumed < dec->limit_ptr) {
 		/* Advance the pointer by the number of full bytes we consumed */
-		dec->cursor += dec->bits_consumed >> 3;
+		dec->cursor += bytes_consumed;
 		/* Refill the bit container */
-		dec->bit_container = bit_read_unalingned_be64(dec->cursor);
-		/* The number of bits that we have already consumed in the current
-		 * byte, excluding the bits that formed a complete byte and were already
-		 * processed.
+		dec->bit_container = bit_read_unaligned_64be(dec->cursor);
+		/* The number of bits that we have already consumed in the
+		 * current byte, excluding the bits that formed a complete byte
+		 * and were already processed.
 		 */
 		dec->bits_consumed &= 0x7;
 		return BIT_UNFINISHED;
 	}
 
-	if (bit_end_of_stream(dec))
+	if (dec->cursor == dec->limit_ptr) {
+		if (dec->bits_consumed == sizeof(dec->bit_container)*8)
 			return BIT_ALL_READ_IN;
-	else
 		return BIT_END_OF_BUFFER;
+	}
 
-	/* limit_ptr < cursor < end */
+	/* limit_ptr < (cursor + bytes_consumed) < end */
 	dec->bits_consumed -= (dec->limit_ptr - dec->cursor)*8;
 	dec->cursor = dec->limit_ptr;
-	dec->bit_container = bit_read_unaligned_64(dec->cursor);
+	dec->bit_container = bit_read_unaligned_64be(dec->cursor);
 
 	return BIT_END_OF_BUFFER;
 }
 
 
+/**
+ * @brief Check if the end of the bitstream has been reached
+ *
+ * @param dec	a bitstream decoding context
+ *
+ * @returns 1 if bit_decoder has _exactly_ reached its end (all bits consumed)
+ */
+
+static __inline unsigned int bit_end_of_stream(const struct bit_decoder *dec)
+{
+	return ((dec->cursor == dec->limit_ptr) &&
+		(dec->bits_consumed == sizeof(dec->bit_container)*8));
+}
+
 #endif /* READ_BITSTREAM_H */

lib/rdcu_compress/cmp_rdcu.c

@@ -649,7 +649,7 @@ int rdcu_compress_data_parallel(const struct cmp_cfg *cfg,
 			return -1;
 
 		/* calculate the need bytes for the bitstream */
-		cmp_size_4byte = ((last_info->cmp_size >> 3) + 3) & ~0x3U;
+		cmp_size_4byte = cmp_bit_to_4byte(last_info->cmp_size);
 
 		/* parallel read compressed data and write input data from sram
 		 * to mirror */

programs/cmp_tool.c

@@ -299,14 +299,15 @@ int main(int argc, char **argv)
 	}
 
 	{
-		const char str[] = "### PLATO Compression/Decompression Tool Version " CMP_TOOL_VERSION " ###\n";
+		static const char str[] = "### PLATO Compression/Decompression Tool Version " CMP_TOOL_VERSION " ###\n";
 		size_t str_len = strlen(str) - 1; /* -1 for \n */
 		size_t i;
-		for (i = 0; i < str_len; ++i)
+
+		for (i = 0; i < str_len; i++)
 			printf("#");
 		printf("\n");
 		printf("%s", str);
-		for (i = 0; i < str_len; ++i)
+		for (i = 0; i < str_len; i++)
 			printf("#");
 		printf("\n");
 	}
@@ -372,7 +373,6 @@ int main(int argc, char **argv)
 		if (info_file_name) {
 			ssize_t f_size;
 			size_t ent_size;
-			uint32_t cmp_size_byte;
 
 			printf("Importing decompression information file %s ... ", info_file_name);
 			error  = cmp_info_read(info_file_name, &info, io_flags & CMP_IO_VERBOSE);
@@ -383,7 +383,7 @@ int main(int argc, char **argv)
 			printf("Importing compressed data file %s ... ", data_file_name);
 
 			ent_size = cmp_ent_create(NULL, DATA_TYPE_IMAGETTE, info.cmp_mode_used == CMP_MODE_RAW,
-						  cmp_bit_to_4byte(info.cmp_size));
+						  cmp_bit_to_byte(info.cmp_size));
 			if (!ent_size)
 				goto fail;
 			decomp_entity = calloc(1, ent_size);
@@ -392,13 +392,12 @@ int main(int argc, char **argv)
 				goto fail;
 			}
 			ent_size = cmp_ent_create(decomp_entity, DATA_TYPE_IMAGETTE, info.cmp_mode_used == CMP_MODE_RAW,
-						  cmp_bit_to_4byte(info.cmp_size));
+						  cmp_bit_to_byte(info.cmp_size));
 			if (!ent_size)
 				goto fail;
 
-			cmp_size_byte = (info.cmp_size+7)/CHAR_BIT;
 			f_size = read_file8(data_file_name, cmp_ent_get_data_buf(decomp_entity),
-					    cmp_size_byte, io_flags);
+					    cmp_bit_to_byte(info.cmp_size), io_flags);
 			if (f_size < 0)
 				goto fail;
 
@@ -417,10 +416,6 @@ int main(int argc, char **argv)
 			buf_size = (size_t)size;
 			if (buf_size < sizeof(struct cmp_entity))
 				buf_size = sizeof(struct cmp_entity);
-			/* The compressed data is read in 4-byte words, so our
-			 * data buffer must be a multiple of 4 bytes.
-			 */
-			buf_size = (buf_size + 3) & ~((size_t)0x3);
 
 			decomp_entity = calloc(1, buf_size);
 			if (!decomp_entity) {
@@ -432,11 +427,6 @@ int main(int argc, char **argv)
 			if (size < 0)
 				goto fail;
 
-			if (cmp_ent_get_size(decomp_entity) & 0x3) {
-				printf("\nThe size of the compression entity is not a multiple of 4 bytes. Padding the compression entity to a multiple of 4 bytes.\n");
-				cmp_ent_set_size(decomp_entity, (uint32_t)buf_size);
-			}
-
 			if (io_flags & CMP_IO_VERBOSE_EXTRA) {
 				cmp_ent_print(decomp_entity);
 				printf("\n");
@@ -600,7 +590,7 @@ static int guess_cmp_pars(struct cmp_cfg *cfg, const char *guess_cmp_mode,
 		return -1;
 
 	if (include_cmp_header)
-		cmp_size_bit = CHAR_BIT * (cmp_bit_to_4byte(cmp_size_bit) +
+		cmp_size_bit = CHAR_BIT * (cmp_bit_to_byte(cmp_size_bit) +
 			cmp_ent_cal_hdr_size(cfg->data_type, cfg->cmp_mode == CMP_MODE_RAW));
 
 	printf("DONE\n");
@@ -768,9 +758,8 @@ static int compression(struct cmp_cfg *cfg, struct cmp_info *info)
 	}
 
 	printf("Compress data ... ");
-	/* round up to a multiple of 4 */
-	out_buf_size = (cmp_cal_size_of_data(cfg->buffer_length, cfg->data_type) + 3) & ~0x3U;
 
+	out_buf_size = cmp_cal_size_of_data(cfg->buffer_length, cfg->data_type);
 	cmp_entity = calloc(1, out_buf_size + sizeof(struct cmp_entity));
 	if (cmp_entity == NULL) {
 		fprintf(stderr, "%s: Error allocating memory for output buffer.\n", PROGRAM_NAME);
@@ -821,10 +810,7 @@ static int compression(struct cmp_cfg *cfg, struct cmp_info *info)
 		if (cmp_gernate_rdcu_info(cfg, cmp_size, ap1_cmp_size, ap2_cmp_size, info))
 			goto error_cleanup;
 		data_to_write_to_file = cmp_ent_get_data_buf(cmp_entity);
-		if (cfg->cmp_mode == CMP_MODE_RAW)
-			cmp_size_byte = info->cmp_size/CHAR_BIT;
-		else
-			cmp_size_byte = cmp_bit_to_4byte(info->cmp_size);
+		cmp_size_byte = cmp_ent_get_cmp_data_size(cmp_entity);
 	}
 
 	printf("DONE\n");

test/cmp_decmp/test_cmp_decmp.c

@@ -513,8 +513,21 @@ void compression_decompression(struct cmp_cfg *cfg)
 	TEST_ASSERT(cmp_size_bits > 0);
 
 	/* put the compression parameters in the entity header */
-	error = cmp_ent_write_cmp_pars(ent, cfg, cmp_size_bits);
+	{
+		/* mock values */
+		uint32_t version_id = ~0U;
+		uint64_t start_time = 32;
+		uint64_t end_time = 42;
+		uint16_t model_id = 0xCAFE;
+		uint8_t model_counter = 0;
+		uint32_t ent_size;
+
+		ent_size = cmp_ent_build(ent, version_id, start_time, end_time,
+					 model_id, model_counter, cfg, cmp_size_bits);
+		TEST_ASSERT_NOT_EQUAL_UINT(0, ent_size);
+		error = cmp_ent_set_size(ent, ent_size);
 		TEST_ASSERT_FALSE(error);
+	}
 
 	/* allocate the buffers for decompression */
 	TEST_ASSERT_NOT_EQUAL_INT(0, data_size);
@@ -604,8 +617,8 @@ void test_random_compression_decompression(void)
 	compression_decompression(NULL);
 }
 
-#define N_SAMPLES 5
 
+#define N_SAMPLES 5
 void test_random_compression_decompression2(void)
 {
 	struct cmp_cfg cfg;

test/cmp_entity/test_cmp_entity.c

@@ -1860,10 +1860,10 @@ void test_cmp_ent_build(void)
 	cmp_size_bits = 2;
 	size = cmp_ent_build(NULL, version_id, start_time, end_time, model_id,
 			     model_counter, &cfg, cmp_size_bits);
-	TEST_ASSERT_EQUAL_UINT(IMAGETTE_ADAPTIVE_HEADER_SIZE+4, size);
+	TEST_ASSERT_EQUAL_UINT(IMAGETTE_ADAPTIVE_HEADER_SIZE+cmp_bit_to_byte((unsigned int)cmp_size_bits), size);
 	size = cmp_ent_build(ent, version_id, start_time, end_time, model_id,
 			     model_counter, &cfg, cmp_size_bits);
-	TEST_ASSERT_EQUAL_UINT(IMAGETTE_ADAPTIVE_HEADER_SIZE+4, size);
+	TEST_ASSERT_EQUAL_UINT(IMAGETTE_ADAPTIVE_HEADER_SIZE+cmp_bit_to_byte((unsigned int)cmp_size_bits), size);
 
 	/** error cases **/
 	/* cfg = NULL */

test/cmp_tool/cmp_tool_integration_test.py

@@ -425,11 +425,11 @@ def test_compression_diff():
                             if arg == " --no_header" else "")(add_arg))
                    )
              # check compressed data
-            cmp_data = "44 44 40 00 \n"
+            cmp_data = "44 44 40 \n"
             with open(output_prefix+".cmp", encoding='utf-8') as f:
                 if add_arg == " --no_header":
                     # check compressed data file
-                    assert(f.read() == "44 44 40 00 \n")
+                    assert(f.read() == cmp_data)
                     # check info file
                     with open(output_prefix+".info", encoding='utf-8') as f:
                         info = parse_key_value(f.read())
@@ -444,7 +444,7 @@ def test_compression_diff():
                 else:
                     header = read_in_cmp_header(f.read())
                     assert(header['asw_version_id']['value'] == VERSION.split('-')[0])
-                    assert(header['cmp_ent_size']['value'] == IMAGETTE_HEADER_SIZE+4)
+                    assert(header['cmp_ent_size']['value'] == IMAGETTE_HEADER_SIZE+3)
                     assert(header['original_size']['value'] == 10)
                     # todo
                     assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow()))
@@ -458,7 +458,7 @@ def test_compression_diff():
                     assert(header['lossy_cmp_par_used']['value'] == 0)
                     assert(header['spill_used']['value'] == 60)
                     assert(header['golomb_par_used']['value'] == 7)
-                    assert(header['compressed_data']['value'] == "44444000")
+                    assert(header['compressed_data']['value'] == "444440")
 
             # decompression
             if add_arg == " --no_header":
@@ -553,7 +553,7 @@ def test_model_compression():
         if "--no_header" in add_arg:
             # check compressed data
             with open(output_prefix1+".cmp", encoding='utf-8') as f:
-                assert(f.read() == "49 24 00 00 \n")
+                assert(f.read() == "49 24 \n")
             # check info file
             with open(output_prefix1+".info", encoding='utf-8') as f:
                 info = parse_key_value(f.read())
@@ -579,7 +579,7 @@ def test_model_compression():
                         header = read_in_cmp_header(bytearray(f.read()).hex())
 
                 assert(header['asw_version_id']['value'] == VERSION.split('-')[0])
-                assert(header['cmp_ent_size']['value'] == IMAGETTE_ADAPTIVE_HEADER_SIZE+4)
+                assert(header['cmp_ent_size']['value'] == IMAGETTE_ADAPTIVE_HEADER_SIZE+2)
                 assert(header['original_size']['value'] == 10)
                 # todo
                 assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow()))
@@ -593,7 +593,7 @@ def test_model_compression():
                 assert(header['lossy_cmp_par_used']['value'] == int(cfg['round']))
                 assert(header['spill_used']['value'] == int(cfg['spill']))
                 assert(header['golomb_par_used']['value'] == int(cfg['golomb_par']))
-                assert(header['compressed_data']['value'] == "49240000")
+                assert(header['compressed_data']['value'] == "4924")
 
             # decompression
             if "--no_header" in add_arg:
@@ -688,7 +688,7 @@ def test_raw_mode_compression():
                 else:
                     header = read_in_cmp_header(f.read())
                     assert(header['asw_version_id']['value'] == VERSION.split('-')[0])
-                    assert(header['cmp_ent_size']['value'] == GENERIC_HEADER_SIZE+12)
+                    assert(header['cmp_ent_size']['value'] == GENERIC_HEADER_SIZE+10)
                     assert(header['original_size']['value'] == 10)
                     # todo
                     assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow()))
@@ -702,7 +702,7 @@ def test_raw_mode_compression():
                     assert(header['lossy_cmp_par_used']['value'] == 0)
                     # assert(header['spill_used']['value'] == 60)
                     # assert(header['golomb_par_used']['value'] == 7)
-                    assert(header['compressed_data']['value'] == data[:-1].replace(" ","")+"0000")
+                    assert(header['compressed_data']['value'] == data[:-1].replace(" ",""))
 
             # decompression
             if "--no_header" in arg:
@@ -768,13 +768,13 @@ def test_guess_option():
                     exp_out = ('', '2', '', '7.27')
                 elif sub_test == 'guess_RDCU_model':
                     exp_out = (
-                        'Importing model file model.dat ... DONE\n', '2', '', str(round((5*2)/(IMAGETTE_ADAPTIVE_HEADER_SIZE + 4), 2)))
-                        #cmp_size:15bit-> 4byte cmp_data + 40byte header -> 16bit*5/(44Byte*8) '5.33'
+                        'Importing model file model.dat ... DONE\n', '2', '', str(round((5*2)/(IMAGETTE_ADAPTIVE_HEADER_SIZE + 2), 2)))
+                        #cmp_size:15bit-> 2byte cmp_data + 40byte header -> 16bit*5/(42Byte*8)
                 elif sub_test == 'guess_level_3':
                     exp_out = (
                         '', '3', ' 0%... 6%... 13%... 19%... 25%... 32%... 38%... 44%... 50%... 57%... 64%... 72%... 80%... 88%... 94%... 100%',
-                        str(round((5*2)/(IMAGETTE_HEADER_SIZE + 4), 2))) #11.43
-                    # cmp_size:7 bit -> 4byte cmp_data + 34 byte header -> 16bit*5/(40Byte*8)
+                        str(round((5*2)/(IMAGETTE_HEADER_SIZE + 1), 3))) #11.43
+                    # cmp_size:7 bit -> 1byte cmp_data + 34 byte header -> 16bit*5/(35Byte*8)
                 else:
                     exp_out = ('', '', '')
 
@@ -1081,7 +1081,7 @@ def test_cmp_entity_not_4_byte_aligned():
 
     version_id = 0x8001_0042
     cmp_ent_size = IMAGETTE_HEADER_SIZE + len(cmp_data)//2
-    original_size = 0xA
+    original_size = 0xC
 
     start_time = cuc_timestamp(datetime.utcnow())
     end_time = cuc_timestamp(datetime.utcnow())
@@ -1103,9 +1103,9 @@ def test_cmp_entity_not_4_byte_aligned():
     ima_header = build_imagette_header(spill_used, golomb_par_used)
     cmp_data_header = generic_header + ima_header + cmp_data
 
-    data_exp = '00 01 00 02 00 03 00 04 00 05 \n'
+    data_exp = '00 01 00 02 00 03 00 04 00 05 00 06 \n'
     info = ("cmp_size = 20\n" + "golomb_par_used = 7\n" + "spill_used = 60\n"
-            + "cmp_mode_used = 2\n" +"samples_used=5\n")
+            + "cmp_mode_used = 2\n" +"samples_used=6\n")
 
     cmp_file_name = 'unaligned_cmp_size.cmp'
     info_file_name = 'unaligned_cmp_size.info'
@@ -1136,9 +1136,7 @@ def test_cmp_entity_not_4_byte_aligned():
                        "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix))
             else:
                 assert(stdout == CMP_START_STR_DECMP +
-                       "Importing compressed data file %s ... \n" % (cmp_file_name) +
-                       "The size of the compression entity is not a multiple of 4 bytes. Padding the compression entity to a multiple of 4 bytes.\n" +
-                       "DONE\n" +
+                       "Importing compressed data file %s ... DONE\n" % (cmp_file_name) +
                        "Decompress data ... DONE\n"+
                        "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix))
 
@@ -1173,7 +1171,7 @@ def test_header_wrong_formatted():
 def test_header_read_in():
     cmp_file_name = 'test_header_read_in.cmp'
 
-    cmp_data = '44444400'
+    cmp_data = '444444'
 
     version_id = 0x8001_0042
     cmp_ent_size = IMAGETTE_HEADER_SIZE + len(cmp_data)//2
@@ -1209,7 +1207,7 @@ def test_header_read_in():
                "Importing compressed data file %s ... FAILED\n" % (cmp_file_name))
         assert(stderr == "cmp_tool: %s: The size of the compression entity set in the "
                "header of the compression entity is not the same size as the read-in "
-               "file has. Expected: 0x28, has 0x26.\n" %(cmp_file_name))
+               "file has. Expected: 0x27, has 0x25.\n" %(cmp_file_name))
 
         # false data type
         data_type = 0x7FFE
@@ -1380,7 +1378,7 @@ def test_rdcu_pkt():
 
             # check compressed data
             with open(output_prefix1+".cmp", encoding='utf-8') as f:
-                assert(f.read() == "49 24 00 00 \n")
+                assert(f.read() == "49 24 \n")
             # check info file
             with open(output_prefix1+".info", encoding='utf-8') as f:
                 info = parse_key_value(f.read())