Refactor: chunk_round_trip() test

lib/common/cmp_max_used_bits.h

@@ -29,6 +29,7 @@
 /* predefined maximum used bits registry constants */
 extern const struct cmp_max_used_bits MAX_USED_BITS_SAFE;
 extern const struct cmp_max_used_bits MAX_USED_BITS_V1;
+#define MAX_USED_BITS MAX_USED_BITS_SAFE
 
 
 /**

test/fuzz/fuzz_compression.c

@@ -26,6 +26,7 @@
 
 #include "fuzz_helpers.h"
 #include "fuzz_data_producer.h"
+#include "../test_common/test_common.h"
 
 #include "../../lib/cmp_chunk.h"
 
@@ -61,7 +62,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 	if (cmp_is_error(cmp_size_bound))
 		cmp_size_bound = 0;
 	cmp_data_capacity = FUZZ_dataProducer_uint32Range(producer, 0, cmp_size_bound+(uint32_t)size);
-	cmp_data = (uint32_t *)FUZZ_malloc(cmp_data_capacity);
+	cmp_data = (uint32_t *)TEST_malloc(cmp_data_capacity);
 
 	FUZZ_dataProducer_cmp_par(producer, &cmp_par);
 	cmp_par.lossy_par = 0; /*TODO: implement lossy  */
@@ -74,10 +75,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 		up_model = NULL;
 		break;
 	case 1:
-		up_model = FUZZ_malloc(size);
+		up_model = TEST_malloc(size);
 		break;
 	case 2:
-		up_model = FUZZ_malloc(size);
+		up_model = TEST_malloc(size);
 		if (model && up_model) {
 			memcpy(up_model, model, size);
 			model = up_model; /* in-place update */

test/fuzz/fuzz_data_producer.c

@@ -23,6 +23,7 @@
 
 #include "fuzz_helpers.h"
 #include "fuzz_data_producer.h"
+#include "../test_common/test_common.h"
 #include <cmp_chunk.h>
 
 struct FUZZ_dataProducer_s{
@@ -31,7 +32,7 @@ struct FUZZ_dataProducer_s{
 };
 
 FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size) {
-    FUZZ_dataProducer_t *producer = FUZZ_malloc(sizeof(FUZZ_dataProducer_t));
+    FUZZ_dataProducer_t *producer = TEST_malloc(sizeof(FUZZ_dataProducer_t));
 
     producer->data = data;
     producer->size = size;

test/fuzz/fuzz_helpers.c

-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE.BSD-3.Zstandard file in the 3rdparty_licenses directory) and the GPLv2
- * (found in the LICENSE.GPL-2 file in the 3rdparty_licenses directory).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#include "fuzz_helpers.h"
-
-#include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-
-void* FUZZ_malloc(size_t size)
-{
-    if (size > 0) {
-        void* const mem = malloc(size);
-        FUZZ_ASSERT(mem);
-        return mem;
-    }
-    return NULL;
-}
-
-void* FUZZ_malloc_rand(size_t size, FUZZ_dataProducer_t *producer)
-{
-    if (size > 0) {
-        void* const mem = malloc(size);
-        FUZZ_ASSERT(mem);
-        return mem;
-    } else {
-        uintptr_t ptr = 0;
-        /* Add +- 1M 50% of the time */
-        if (FUZZ_dataProducer_uint32Range(producer, 0, 1))
-            FUZZ_dataProducer_int32Range(producer, -1000000, 1000000);
-        return (void*)ptr;
-    }
-
-}
-
-int FUZZ_memcmp(void const* lhs, void const* rhs, int32_t size)
-{
-    if (size <= 0) {
-        return 0;
-    }
-    return memcmp(lhs, rhs, (size_t)size);
-}

test/fuzz/fuzz_helpers.h

@@ -15,18 +15,12 @@
 #ifndef FUZZ_HELPERS_H
 #define FUZZ_HELPERS_H
 
-#include "fuzz.h"
-#include "fuzz_data_producer.h"
-#include <stdint.h>
 #include <stdio.h>
-#include <stdlib.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
 
 #define FUZZ_QUOTE_IMPL(str) #str
 #define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str)
@@ -40,36 +34,6 @@ extern "C" {
                      __LINE__, FUZZ_QUOTE(cond), (msg)),                       \
              abort()))
 #define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), "");
-#define FUZZ_ZASSERT(code)                                                     \
-  FUZZ_ASSERT_MSG(!ZSTD_isError(code), ZSTD_getErrorName(code))
-
-#if defined(__GNUC__)
-#define FUZZ_STATIC static __inline __attribute__((unused))
-#elif defined(__cplusplus) ||                                                  \
-    (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-#define FUZZ_STATIC static inline
-#elif defined(_MSC_VER)
-#define FUZZ_STATIC static __inline
-#else
-#define FUZZ_STATIC static
-#endif
-
-/**
- * malloc except return NULL for zero sized data and FUZZ_ASSERT
- * that malloc doesn't fail.
- */
-void* FUZZ_malloc(size_t size);
-
-/**
- * malloc except returns random pointer for zero sized data and FUZZ_ASSERT
- * that malloc doesn't fail.
- */
-void* FUZZ_malloc_rand(size_t size,  FUZZ_dataProducer_t *producer);
-
-/**
- * memcmp but accepts NULL. Ignore negative sizes
- */
-int FUZZ_memcmp(void const* lhs, void const* rhs, int32_t size);
 
 #ifdef __cplusplus
 }

test/fuzz/fuzz_round_trip.c

@@ -22,104 +22,12 @@
  */
 
 #include <string.h>
+#include <stdlib.h>
 
 #include "fuzz_helpers.h"
 #include "fuzz_data_producer.h"
-
-#include "../../lib/cmp_chunk.h"
-#include "../../lib/decmp.h"
-
-
-#define TEST_malloc(size) FUZZ_malloc(size)
-#define TEST_ASSERT(cond) FUZZ_ASSERT(cond)
-
-
-static uint32_t chunk_round_trip(void *data, uint32_t data_size,
-				 void *model, void *up_model,
-				 uint32_t *cmp_data, uint32_t cmp_data_capacity,
-				 struct cmp_par *cmp_par, int use_decmp_buf,
-				 int use_decmp_up_model)
-{
-	uint32_t cmp_size;
-	void *model_cpy = NULL;
-
-	/* if in-place model update is used (up_model == model), the model
-	 * needed for decompression is destroyed; therefore we make a copy
-	 */
-	if (model) {
-		if (up_model == model) {
-			model_cpy = TEST_malloc(data_size);
-			memcpy(model_cpy, model, data_size);
-		} else {
-			model_cpy = model;
-		}
-	}
-
-	cmp_size = compress_chunk(data, data_size, model, up_model,
-				  cmp_data, cmp_data_capacity, cmp_par);
-
-#if 0
-	{ /* Compress a second time and check for determinism */
-		int32_t cSize2;
-		void *compressed2 = NULL;
-		void *up_model2 = NULL;
-
-		if (compressed)
-			compressed2 = FUZZ_malloc(compressedCapacity);
-
-		if (up_model)
-			up_model2 = FUZZ_malloc(srcSize);
-		cSize2 = compress_chunk((void *)src, srcSize, (void *)model, up_model2,
-					   compressed2, compressedCapacity, cmp_par);
-		FUZZ_ASSERT(cSize == cSize2);
-		FUZZ_ASSERT_MSG(!FUZZ_memcmp(compressed, compressed2, cSize), "Not deterministic!");
-		FUZZ_ASSERT_MSG(!FUZZ_memcmp(up_model, compressed2, cSize), "NO deterministic!");
-		free(compressed2);
-		free(up_model2);
-	}
-#endif
-	if (!cmp_is_error(cmp_size) && cmp_data) {
-		void *decmp_data = NULL;
-		void *up_model_decmp = NULL;
-		int decmp_size;
-
-		decmp_size = decompress_cmp_entiy((struct cmp_entity *)cmp_data, model_cpy, NULL, NULL);
-		TEST_ASSERT(decmp_size >= 0);
-		TEST_ASSERT((uint32_t)decmp_size == data_size);
-
-		if (use_decmp_buf)
-			decmp_data = TEST_malloc(data_size);
-		if (use_decmp_up_model) {
-			up_model_decmp = TEST_malloc(data_size);
-			if (!model_mode_is_used(cmp_par->cmp_mode))
-				memset(up_model_decmp, 0, data_size); /* up_model is not used */
-		}
-
-		decmp_size = decompress_cmp_entiy((struct cmp_entity *)cmp_data, model_cpy,
-						  up_model_decmp, decmp_data);
-		TEST_ASSERT(decmp_size >= 0);
-		TEST_ASSERT((uint32_t)decmp_size == data_size);
-
-		if (use_decmp_buf) {
-			TEST_ASSERT(!memcmp(data, decmp_data, data_size));
-
-			/*
-			 * the model is only updated when the decompressed_data
-			 * buffer is set
-			 */
-			if (up_model && up_model_decmp)
-				TEST_ASSERT(!memcmp(up_model, up_model_decmp, data_size));
-		}
-
-		free(decmp_data);
-		free(up_model_decmp);
-	}
-
-	if (up_model == model)
-		free(model_cpy);
-
-	return cmp_size;
-}
+#include "../test_common/chunk_round_trip.h"
+#include "../test_common/test_common.h"
 
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
@@ -155,7 +63,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
 	/* 1/2 of the cases we use a updated model buffer */
 	if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) {
-		up_model = FUZZ_malloc(size);
+		up_model = TEST_malloc(size);
 		if (!model_mode_is_used(cmp_par.cmp_mode))
 			memset(up_model, 0, size); /* up_model is not used */
 	}
@@ -164,7 +72,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 	if (cmp_is_error(cmp_size_bound))
 		cmp_size_bound = 0;
 	cmp_data_capacity = FUZZ_dataProducer_uint32Range(producer, 0, cmp_size_bound+(uint32_t)size);
-	cmp_data = (uint32_t *)FUZZ_malloc(cmp_data_capacity);
+	cmp_data = (uint32_t *)TEST_malloc(cmp_data_capacity);
 
 	use_decmp_buf = FUZZ_dataProducer_int32Range(producer, 0, 1);
 	use_decmp_up_model = FUZZ_dataProducer_int32Range(producer, 0, 1);

test/fuzz/meson.build

@@ -2,7 +2,7 @@ if get_option('fuzzer').disabled()
   subdir_done()
 endif
 
-fuzz_common = files('fuzz_helpers.c', 'fuzz_data_producer.c')
+fuzz_common = files('fuzz_data_producer.c')
 fuzz_targets = ['fuzz_compression.c', 'fuzz_round_trip.c']
 
 add_languages('cpp', native: false) # libFuzzingEngine needs c++
@@ -14,7 +14,7 @@ foreach target : fuzz_targets
   fuzz_exe = executable(target_name,
     fuzz_common, file_name,
     include_directories : incdir,
-    link_with : cmp_lib,
+    link_with : [cmp_lib, test_common_lib],
     link_args : get_option('fuzzer_ldflags'),
     link_language : 'cpp' # libFuzzingEngine needs c++
   )

test/meson.build

@@ -53,7 +53,7 @@ if cc.has_argument('-fsanitize=leak') and not (host_machine.system() == 'darwin'
 endif
 
 test_env.set('UBSAN_OPTIONS',
-  'abort_on_error=1',
+  'halt_on_error=1',
   'print_stacktrace=1',
   'print_summary=1',
   'symbolize=1',
@@ -71,6 +71,7 @@ subdir('tools')
 subdir('cmp_tool')
 
 unity_dep = dependency('unity', fallback : ['unity', 'unity_dep'])
+m_dep = cc.find_library('m', required : false)
 subdir('test_common')
 subdir('fuzz')
 
@@ -104,7 +105,7 @@ if ruby.found()
       test_src, test_runner,
       include_directories : incdir,
       link_with : test_libs,
-      dependencies : unity_dep,
+      dependencies : [unity_dep, m_dep],
       c_args : test_args,
       build_by_default : false
     )

test/test_common/chunk_round_trip.c

+/**
+ * @file chunk_round_trip.c
+ * @date 2024
+ *
+ * @copyright GPLv2
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * @brief chunk compression/decompression test
+ *
+ */
+
+#include <string.h>
+#include <stdlib.h>
+
+#include "../test_common/test_common.h"
+#include "../../lib/cmp_chunk.h"
+#include "../../lib/decmp.h"
+#include "chunk_round_trip.h"
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+#  include "../fuzz/fuzz_helpers.h"
+#  define TEST_ASSERT(cond) FUZZ_ASSERT(cond)
+#else
+#  include <unity.h>
+#endif
+
+
+/**
+ * @brief performs chunk compression and checks if a decompression is possible
+ *
+ * @param chunk			pointer to the chunk to be compressed
+ * @param chunk_size		byte size of the chunk
+ * @param chunk_model		pointer to a model of a chunk; has the same size
+ *				as the chunk (can be NULL if no model compression
+ *				mode is used)
+ * @param updated_chunk_model	pointer to store the updated model for the next
+ *				model mode compression; has the same size as the
+ *				chunk (can be the same as the model_of_data
+ *				buffer for in-place update or NULL if updated
+ *				model is not needed)
+ * @param dst			destination pointer to the compressed data
+ *				buffer; has to be 4-byte aligned; can be NULL to
+ *				only get the compressed data size
+ * @param dst_capacity		capacity of the dst buffer; it's recommended to
+ *				provide a dst_capacity >=
+ *				compress_chunk_cmp_size_bound(chunk, chunk_size)
+ *				as it eliminates one potential failure scenario:
+ *				not enough space in the dst buffer to write the
+ *				compressed data; size is internally rounded down
+ *				to a multiple of 4
+ * @param use_decmp_buf		if non-zero, a buffer is allocated for the
+ *				decompressed data
+ * @param use_decmp_up_model	if non-zero, a buffer for the updated model is
+ *				allocated during the decompression
+ *
+ * @returns the return value of the compress_chunk() function
+ */
+
+uint32_t chunk_round_trip(void *chunk, uint32_t chunk_size,
+			  void *chunk_model, void *updated_chunk_model,
+			  uint32_t *dst, uint32_t dst_capacity,
+			  struct cmp_par *cmp_par,
+			  int use_decmp_buf, int use_decmp_up_model)
+{
+	uint32_t cmp_size;
+	void *model_cpy = NULL;
+
+	/* if in-place model update is used (up_model == model), the model
+	 * needed for decompression is destroyed; therefore we make a copy
+	 */
+	if (chunk_model) {
+		if (updated_chunk_model == chunk_model) {
+			model_cpy = TEST_malloc(chunk_size);
+			memcpy(model_cpy, chunk_model, chunk_size);
+		} else {
+			model_cpy = chunk_model;
+		}
+	}
+
+	cmp_size = compress_chunk(chunk, chunk_size, chunk_model, updated_chunk_model,
+				  dst, dst_capacity, cmp_par);
+
+
+	/* decompress the data if the compression was successful */
+	if (!cmp_is_error(cmp_size) && dst) {
+		void *decmp_data = NULL;
+		void *up_model_decmp = NULL;
+		int decmp_size;
+
+		decmp_size = decompress_cmp_entiy((struct cmp_entity *)dst, model_cpy, NULL, NULL);
+		TEST_ASSERT(decmp_size >= 0);
+		TEST_ASSERT((uint32_t)decmp_size == chunk_size);
+
+		if (use_decmp_buf)
+			decmp_data = TEST_malloc(chunk_size);
+		if (use_decmp_up_model) {
+			up_model_decmp = TEST_malloc(chunk_size);
+			if (!model_mode_is_used(cmp_par->cmp_mode))
+				memset(up_model_decmp, 0, chunk_size); /* up_model is not used */
+		}
+
+		decmp_size = decompress_cmp_entiy((struct cmp_entity *)dst, model_cpy,
+						  up_model_decmp, decmp_data);
+		TEST_ASSERT(decmp_size >= 0);
+		TEST_ASSERT((uint32_t)decmp_size == chunk_size);
+
+		if (use_decmp_buf) {
+			TEST_ASSERT(!memcmp(chunk, decmp_data, chunk_size));
+
+			/*
+			 * the model is only updated when the decompressed_data
+			 * buffer is set
+			 */
+			if (use_decmp_up_model && updated_chunk_model)
+				TEST_ASSERT(!memcmp(updated_chunk_model, up_model_decmp, chunk_size));
+		}
+
+		free(decmp_data);
+		free(up_model_decmp);
+	}
+
+	if (updated_chunk_model == chunk_model)
+		free(model_cpy);
+
+	return cmp_size;
+}

test/test_common/chunk_round_trip.h

+/**
+ * @file chunk_round_trip.h
+ * @date 2024
+ *
+ * @copyright GPLv2
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * @brief chunk compression/decompression
+ */
+
+
+#ifndef CHUNK_ROUND_TRIP_H
+#define CHUNK_ROUND_TRIP_H
+
+#include <stdint.h>
+#include "../../lib/cmp_chunk.h"
+
+
+uint32_t chunk_round_trip(void *data, uint32_t data_size,
+			  void *model, void *up_model,
+			  uint32_t *cmp_data, uint32_t cmp_data_capacity,
+			  struct cmp_par *cmp_par, int use_decmp_buf, int
+			  use_decmp_up_model);
+
+
+#endif /* CHUNK_ROUND_TRIP_H */
+

test/test_common/meson.build

@@ -4,5 +4,6 @@ pcb_dep = pcg_proj.get_variable('libpcg_basic_dep')
 test_common_lib = static_library(
    'test_common',
    'test_common.c',
+   'chunk_round_trip.c',
    dependencies: [pcb_dep, unity_dep]
 )

test/test_common/test_common.c

@@ -6,7 +6,12 @@
 #include "pcg_basic.h"
 #include "test_common.h"
 
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+#  include "../fuzz/fuzz_helpers.h"
+#  define TEST_ASSERT(cond) FUZZ_ASSERT(cond)
+#else
 #  include <unity.h>
+#endif
 
 void cmp_rand_seed(uint64_t seed)
 {
@@ -38,11 +43,12 @@ uint32_t cmp_rand_between(uint32_t min, uint32_t max)
 }
 
 
-uint32_t cmp_rand_nbits(unsigned int nbits)
+uint32_t cmp_rand_nbits(unsigned int n_bits)
 {
-	TEST_ASSERT(nbits > 0);
+	TEST_ASSERT(n_bits > 0);
+	TEST_ASSERT(n_bits <= 32);
 
-	return cmp_rand32() >> (32 - nbits);
+	return cmp_rand32() >> (32 - n_bits);
 }
 
 

test/test_common/test_common.h

@@ -10,7 +10,7 @@ uint32_t cmp_rand32(void);
 
 uint32_t cmp_rand_between(uint32_t min, uint32_t max);
 
-uint32_t cmp_rand_nbits(unsigned int nbits);
+uint32_t cmp_rand_nbits(unsigned int n_bits);
 
 void* TEST_malloc(size_t size);