From 92ae9aab046c3ece46cc798426c040947adccb2b Mon Sep 17 00:00:00 2001 From: Evgeny Voropaev Date: Fri, 20 Mar 2026 17:30:22 +0800 Subject: [PATCH v09 2/3] Implement Delta Frame of Reference compression. Implement the compression algorithm based on the Delta Frame of Reference technique (DFOR). DFoR supports both external memory (outer memory) provided by a caller and automatically managed memory, allocated by means of malloc, palloc or similar functions. Memory management configuration must be defined during initialization. All subsequent operations follow this configuration. For example, a caller can place a buffer on the stack to avoid heap allocation and pass the buffer to a DFoR unit. As a result, the packing and unpacking processes exclude dynamic allocation. The DFoR unit is implemented as a set of templates. Developers can generate DFoR implementations for any unsigned integer type (uint8_t, uint16_t, uint32_t, uint64_t). The dfor_u16 unit is implemented. The unit test is implemented as a C program (ELF executable). The test can be run with the 'make check-unit'. Tests support the TAP protocol and are executed using the Prove utility. Author: Evgeny Voropaev Reviewed by: Andrey Borodin --- src/backend/lib/Makefile.dfor | 1 + src/backend/lib/dfor_templ.c | 618 ++++++++++++++++++++++++++++ src/backend/lib/dfor_u16.c | 8 + src/backend/lib/meson.build | 1 + src/include/lib/dfor_templ.h | 27 ++ src/include/lib/dfor_templ_staple.h | 125 ++++++ src/include/lib/dfor_templ_undef.h | 29 ++ src/include/lib/dfor_u16.h | 13 + src/include/lib/dfor_u16_config.h | 4 + src/test/dfor/.gitignore | 1 + src/test/dfor/Makefile | 3 +- src/test/dfor/meson.build | 2 + src/test/dfor/test_dfor_u16.c | 371 +++++++++++++++++ 13 files changed, 1202 insertions(+), 1 deletion(-) create mode 100644 src/backend/lib/dfor_templ.c create mode 100644 src/backend/lib/dfor_u16.c create mode 100644 src/include/lib/dfor_templ.h create mode 100644 src/include/lib/dfor_templ_staple.h create mode 100644 src/include/lib/dfor_templ_undef.h create mode 100644 src/include/lib/dfor_u16.h create mode 100644 src/include/lib/dfor_u16_config.h create mode 100644 src/test/dfor/test_dfor_u16.c diff --git a/src/backend/lib/Makefile.dfor b/src/backend/lib/Makefile.dfor index b93c6e78644..beb7035f155 100644 --- a/src/backend/lib/Makefile.dfor +++ b/src/backend/lib/Makefile.dfor @@ -2,4 +2,5 @@ OBJS_DFOR := \ bitpack_u16.o \ + dfor_u16.o \ vect_u16.o diff --git a/src/backend/lib/dfor_templ.c b/src/backend/lib/dfor_templ.c new file mode 100644 index 00000000000..dae79adfb66 --- /dev/null +++ b/src/backend/lib/dfor_templ.c @@ -0,0 +1,618 @@ +/* + * dfor.c + * + * DFOR_TEMPL implements the variant of the Frame of Reference with Delta + * container and corresponding algorithms. + * + * The type of original items is defined with the item_t macro. item_t must be + * an unsigned integer (uint8_t, uint16_t, ... uint64_t) + * + * Each bit vector, having been serialised, represents next structure: + * + * | deltas | exceptions | exceptions positions | + * + * The delta is a difference between the current item and the previous one. + * The delta of the first item (the member having the zero index) is its actual + * value: delta[0] = m[0]-0 = m[0]. A having serialised delta is a sequence of + * bits. Each serialised delta in 'deltas' has a fixed bit width. If the delta's + * width exceeds the allowed size of a delta in 'deltas', the higher bits of this + * delta is put into exceptions. + */ + +#include "lib/dfor_templ_staple.h" + +int dfor_calc_deltas(size_t cnt, const item_t arr[], vect_t *vDeltas, + uniqsortvect_t *usvDeltaWidths, + vect_t *vWidthCounters); + +int dfor_calc_width(size_t cntDelta, + const uniqsortvect_t *usvDeltaWidths, + const vect_t *vWidthCounters, size_t *width, + size_t *cntExceptions); + +int dfor_analyze(size_t cnt, const item_t arr[], + dfor_meta_t *dfor, vect_t *vDeltas, + uniqsortvect_t *usvExcPos); + +int dfor_pack(size_t cnt, const item_t arr[], excalg_t isExcUsage, + dfor_meta_t *dfor, size_t bufSize, uint8_t buf[]); + +int dfor_unpack(const dfor_meta_t *dfor, uniqsortvect_t *vVals, size_t bufSize, + uint8_t buf[]); + +void dfor_clear_meta(dfor_meta_t *dfor); + +dfor_stats_t dfor_calc_stats(dfor_meta_t dfor); + +size_t dfor_calc_nbytes(dfor_meta_t dfor); + +/* + * Calculate deltas + * + * vWidthCounters being equal to NULL means 'Don't calculate counts of widths'. + * In this case usvDeltaWidth comprise only one member m[0] which saves max + * width of delta, which can be used by caller. + * + */ +int +dfor_calc_deltas(size_t cnt, const item_t arr[], vect_t *vDeltas, + uniqsortvect_t *usvDeltaWidths, vect_t *vWidthCounters) +{ + item_t delta; + item_t prev; /* value of previous number*/ + size_t width; + usv_ins_res_t insWidthInsert; + + if (vDeltas == NULL) + return -1; + + if (vWidthCounters == NULL) + usv_insert(usvDeltaWidths, 0); + + prev = 0; + for (size_t j = 0; j < cnt; j++) { + delta = arr[j] - prev; + vect_append(vDeltas, delta); + prev = arr[j]; + width = width_from_val(delta); + + if (vWidthCounters == NULL) { + if (usvDeltaWidths->m[0] < width) + usvDeltaWidths->m[0] = width; + } else { + insWidthInsert = usv_insert(usvDeltaWidths, width_from_val(delta)); + + if (insWidthInsert.st == USV_INS_NEW) + vect_insert(vWidthCounters, insWidthInsert.pos, (item_t)1); + else if (insWidthInsert.st == USV_INS_EXISTS) + vWidthCounters->m[insWidthInsert.pos]++; + else + return -1; + } + } + return 0; +} + +/* + * Calculate width of short deltas, width of exceptions, and number of + * exceptions + */ +int +dfor_calc_width(size_t cntDelta, const uniqsortvect_t *usvDeltaWidths, + const vect_t *vWidthCounters, size_t *width, + size_t *cntExceptions) +{ +#define MIN_DELTAS_QUANTITY_ALLOWING_EXCEPTIONS 4 + + size_t cntShortDeltas; /* number of deltas presented without exceptions */ + size_t indxWidth; /* the width of short deltas (index from vWidthCounters + * (and from vDeltaWidth accordingly) + */ + if (usvDeltaWidths == NULL || vWidthCounters == NULL || width == NULL || + cntExceptions == NULL) + return -1; + + cntShortDeltas = cntDelta; + indxWidth = usvDeltaWidths->cnt - 1; /* counter into index */ + *cntExceptions = 0; + + /* + * Here we try to decrease the width of short deltas in order to compress + * the array of deltas in the meantime we are eager to cover no less than + * 90% of deltas we have. It is an heuristic analysis based on the + * suggestion "no less than 90% of deltas we have". + * + * TODO: analyzing we might want calulate the full size of the pack for each + * variant of the width. + */ + if (cntDelta >= MIN_DELTAS_QUANTITY_ALLOWING_EXCEPTIONS) { + size_t szMinCoverage; /* threshold */ + size_t j; + + if (cntDelta >= 10) + szMinCoverage = cntDelta - cntDelta / 10; + else + szMinCoverage = cntDelta - 1; + + j = indxWidth; + + while (j > 0) { + if (cntShortDeltas - vWidthCounters->m[j] < szMinCoverage) + break; + + cntShortDeltas -= vWidthCounters->m[j]; + j--; + indxWidth = j; + } + *cntExceptions = cntDelta - cntShortDeltas; + } + + *width = usvDeltaWidths->m[indxWidth]; + return 0; +} + +/* + * dfor_analyze + * Analyze input array, calculate deltas and their width, define exceptions and + * their positions. Returns them through the dfor, vDeltas, usvExcPos. If + * usvExcPos == NULL - don't calculate exceptions. + * + * dfor_analyze function does not use dynamic memory allocation for its + * local containers. + * + * A caller has to control whether vDeltas and usvExcPos use outer memory + * provided by caller or manage memory allocation automatically, which defines + * whether vect_insert and vect_append functions, invoked from here, use dynamic + * memory or not. + * + * A caller should take into account that dfor_meta_t dfor are going to be + * nullified in this function, so it should not have any meaningfull data by + * start of dfor_analyze, especially its pack field should not be used as a + * pointer on dynamic memory, otherwise memory leakage is possible. + * + */ +int +dfor_analyze(size_t cnt, const item_t arr[], /* input */ + dfor_meta_t *dfor, vect_t *vDeltas, + uniqsortvect_t *usvExcPos) /* output */ +{ +#define DELTA_WIDTH_MAX_NUMBER (sizeof(item_t) * 8) + uniqsortvect_t usvDeltaWidths; + item_t bufDeltaWidth[DELTA_WIDTH_MAX_NUMBER]; + vect_t vWidthCounters; + item_t bufWidthCounters[DELTA_WIDTH_MAX_NUMBER]; + item_t mask; + int res = -1; + + excalg_t isExcUsage = (usvExcPos == NULL) ? DFOR_EXC_DONT_USE : + DFOR_EXC_USE; + + if (dfor == NULL) + goto dfor_analyze_error; + + memset(dfor, 0, sizeof(dfor_meta_t)); + + if (cnt == 0) + /* dfor->item_cnt = 0; */ /* it's been already done with memset */ + goto dfor_analyze_ret; + else if (arr == NULL) + goto dfor_analyze_error; + + if (0 != vect_init(&usvDeltaWidths, DELTA_WIDTH_MAX_NUMBER, bufDeltaWidth)) + goto dfor_analyze_error; + + if (isExcUsage == DFOR_EXC_USE) + { + if (0 != + vect_init(&vWidthCounters, DELTA_WIDTH_MAX_NUMBER, + bufWidthCounters)) + goto dfor_analyze_error; + } + + dfor->item_cnt = cnt; + + if (0 != + dfor_calc_deltas(dfor->item_cnt, arr, vDeltas, &usvDeltaWidths, + (isExcUsage == DFOR_EXC_USE) ? &vWidthCounters : NULL)) + goto dfor_analyze_error; + + Assert(cnt == vDeltas->cnt); + Assert(usvDeltaWidths.cnt > 0); + + if (isExcUsage == DFOR_EXC_USE) + { + if (0 != + dfor_calc_width(vDeltas->cnt, &usvDeltaWidths, &vWidthCounters, + &dfor->delta_wid, &dfor->exc_cnt)) + goto dfor_analyze_error; + } + else + { + dfor->delta_wid = + usvDeltaWidths.m[usvDeltaWidths.cnt - 1]; /* max width */ + dfor->exc_cnt = 0; + } + + dfor->exc_wid = usvDeltaWidths.m[usvDeltaWidths.cnt - 1] - dfor->delta_wid; + + /* A mask looks like 0001111. It is also the max value of a short delta */ + mask = width_to_mask(dfor->delta_wid); + + for (size_t i = 0; i < vDeltas->cnt; i++) + { + if (vDeltas->m[i] > mask) + { + Assert(isExcUsage == DFOR_EXC_USE); + if (0 != vect_append(usvExcPos, (item_t)i)) + goto dfor_analyze_error; + } + } + Assert(dfor->delta_wid + dfor->exc_wid <= sizeof(item_t) * 8); + res = 0; +dfor_analyze_ret: + return res; +dfor_analyze_error: + /* dfor_analyze doesn't affect the pack field (doesn't allocate, delete or + * otherwise), so we can nullify the whole dfor and it + * is safe, no leakage */ + memset(dfor, 0, sizeof(dfor_meta_t)); + res = -1; + goto dfor_analyze_ret; +} + +/* + * dfor_pack + * + * The input array arr has to be sorted. + * + * If a caller needs to avoid using dynamic memory allocation, they have to + * provide the external memory buffer. The size of this buffer should be not + * less than 4 * cnt * sizeof(item_t). It will be used for arrays pointed by + * *(dfor->pack), *(vDeltas->m), *(vExcPosDeltas->m), *(usvExcPos->m). + * + * If dynamic allocation has been used by the dfor_pack, a caller has to free + * the piece of memory pointed by dfor->pack, since it is alocated by the + * dfor_pack with DFOR_MALLOC. Freeing has to be performed by function + * conforming to DFOR_MALLOC (paired with it). For instance, if DFOR_MALLOC is + * malloc, than memory should be freed by free. + */ +int +dfor_pack(size_t cnt, const item_t arr[], excalg_t isExcUsage, + dfor_meta_t *dfor, size_t bufSize, uint8_t buf[]) +{ + int res; + vect_t vDeltas = { 0 }; + vect_t vExcPosDeltas = { 0 }; + uniqsortvect_t usvExcPos = { 0 }; + + if (dfor == NULL || + (bufSize != 0 && bufSize < 4 * cnt * sizeof(item_t))) + { + goto dfor_pack_error; + } + + /* + * We don't need it here: + * memset(dfor, 0, sizeof(dfor_meta_t)). + * It is going to be done in dfor_analyze. + */ + + { + item_t *deltaBuf = NULL; + item_t *excPosDeltasBuf = NULL; + item_t *excPosBuf = NULL; + + int res1 = 0; + int res2 = 0; + int res3 = 0; + + if (bufSize != 0) + { + /* Step over the maximal allowed DFoR pack size */ + deltaBuf = (item_t*)(buf + cnt * sizeof(item_t)); + excPosDeltasBuf = (item_t*)(buf + cnt * sizeof(item_t) * 2); + excPosBuf = (item_t*)(buf + cnt * sizeof(item_t) * 3); + } + + /* Setup containers with outer memory */ + res1 = vect_init(&vDeltas, cnt, deltaBuf); + + if (isExcUsage) + { + res2 = vect_init(&vExcPosDeltas, cnt, excPosDeltasBuf); + res3 = vect_init(&usvExcPos, cnt, excPosBuf); + } + + if (res1 != 0 || res2 != 0 || res3 != 0) + goto dfor_pack_error; + } + + if (0 != + dfor_analyze(cnt, arr, dfor, &vDeltas, + (isExcUsage == DFOR_EXC_USE) ? &usvExcPos : NULL)) + goto dfor_pack_error; + + if (dfor->exc_cnt != 0) + { + /* We treat exception positions as a sorted sequence, apply the + * DFoR algorithm to it, and save not their absolute values but their + * deltas. */ + dfor_meta_t dforExcPos; + Assert(dfor->exc_cnt == usvExcPos.cnt); + if (0 != + dfor_analyze(usvExcPos.cnt, usvExcPos.m, &dforExcPos, + &vExcPosDeltas, NULL)) + goto dfor_pack_error; + + Assert(dfor->exc_cnt == vExcPosDeltas.cnt); + Assert(dfor->exc_cnt == dforExcPos.item_cnt); + + dfor->exc_pos_wid = dforExcPos.delta_wid; + } + else + { + Assert(usvExcPos.cnt == 0); /* usvExcPos has to remain zeroed. */ + Assert(dfor->exc_wid == 0); /* No exceptions, no exceptions' width. */ + Assert(dfor->exc_pos_wid == 0); /* No exceptions' positions width too. */ + } + + /* dfor_pack serialisation packing */ + { + /* index of the next free bit to be used: */ + size_t d; /* - by a delta */ + size_t e; /* - by an exception */ + size_t p; /* - by an exception position */ + item_t mask; + dfor_stats_t stats; + size_t j; + + stats = dfor_calc_stats(*dfor); + dfor->nbytes = dfor_calc_nbytes(*dfor); + + if (bufSize != 0) + { + /* Max size of the dfor->pack is cnt * sizeof(size_t) */ + dfor->pack = buf; + dfor->outer_mem = true; + if (dfor->nbytes > cnt * sizeof(size_t)) + goto dfor_pack_error; + } + else + { + /* If a buffer was not provided by caller we allocate it by + * ourselves + */ + dfor->pack = (uint8_t *)DFOR_MALLOC((dfor->nbytes)); + + dfor->outer_mem = false; + } + + if (dfor->pack == NULL) + goto dfor_pack_error; + + memset(dfor->pack, 0, dfor->nbytes); + + /* index of the next free bit to be used: */ + d = 0; /* - by a delta */ + e = stats.delta_pack_nbits; /* - by an exception */ + p = e + stats.exc_pack_nbits; /* - by an exception position index */ + /* A mask looks like 0001111. It is also the + * max value of a short delta */ + mask = width_to_mask(dfor->delta_wid); + + j = 0; + for (size_t i = 0; i < vDeltas.cnt; i++) + { + d = bitpack_pack(dfor->pack, d, vDeltas.m[i] & mask, + dfor->delta_wid); + + if (vDeltas.m[i] > mask) + { + Assert(isExcUsage == DFOR_EXC_USE); + Assert(usvExcPos.m[j] == i); + Assert(j < usvExcPos.cnt); + Assert(j < vExcPosDeltas.cnt); + Assert(dfor->exc_wid != 0); + Assert(dfor->exc_pos_wid != 0); + + e = bitpack_pack(dfor->pack, e, vDeltas.m[i] >> dfor->delta_wid, + dfor->exc_wid); + p = bitpack_pack(dfor->pack, p, vExcPosDeltas.m[j], dfor->exc_pos_wid); + j++; + } + } + + if (isExcUsage == DFOR_EXC_USE) + Assert(j == usvExcPos.cnt); + else + Assert(j == 0); + + Assert(d == stats.delta_pack_nbits); + Assert(e == stats.delta_pack_nbits + stats.exc_pack_nbits); + Assert(p == + stats.delta_pack_nbits + stats.exc_pack_nbits + + stats.exc_pos_pack_nbits); + res = 0; + } +dfor_pack_ret: + vect_clear(&usvExcPos); + vect_clear(&vExcPosDeltas); + vect_clear(&vDeltas); + return res; +dfor_pack_error: + if (dfor != NULL) + dfor_clear_meta(dfor); + res = -1; + goto dfor_pack_ret; +} + +/* + * dfor_unpack + * + * If a caller needs to avoid using dynamic memory allocation, they have to: + * 1) provide the external memory buffer. The size of this buffer should be not + * less than: + * 2 * dfor.item_cnt * sizeof(item_t) + 2 * dfor.exc_cnt * sizeof(item_t) + * + * 2) the vVals vector has to be created but must not be initialised. The + * dfor_unpack sets vVals in the 'outer memory' regimen and will set vVal->m + * to buf. + * + * Provided dynamic allocation is used by the dfor_unpack, a caller will have to + * free the piece of memory pointed by vVals->m, using vect_clear(&vVals). + * + * Are the outer memory is used + */ +int +dfor_unpack(const dfor_meta_t *dfor, uniqsortvect_t *vVals, size_t bufSize, + uint8_t buf[]) +{ + int res = -1; + size_t szDeltaPack; + vect_t vExcs = { 0 }; + vect_t vExcPoss = { 0 }; + excalg_t isExcUsage = (dfor->exc_cnt == 0) ? DFOR_EXC_DONT_USE : + DFOR_EXC_USE; + + if (vVals == NULL) + goto dfor_unpack_error; + + if (bufSize != 0 && + bufSize < (2 * dfor->item_cnt * sizeof(item_t) + + 2 * dfor->exc_cnt * sizeof(item_t))) + goto dfor_unpack_error; + + szDeltaPack = dfor->delta_wid * dfor->item_cnt; + + { + uint8_t *valsBuf = NULL; + if (bufSize != 0) + valsBuf = buf; + + if (vect_init(vVals, dfor->item_cnt, (item_t *)valsBuf) != 0) + goto dfor_unpack_error; + } + + /* Calculate exceptions */ + if (isExcUsage == DFOR_EXC_USE) + { + size_t szExcPack; + size_t crExc; /* caret (cursor) */ + size_t crPos; /* caret (cursor) */ + + uint8_t *excBuf = NULL; + uint8_t *excPossBuf = NULL; + + int res1 = 0; + int res2 = 0; + + szExcPack = dfor->exc_cnt * dfor->exc_wid; + crExc = szDeltaPack; + crPos = crExc + szExcPack; + + if (bufSize != 0) + { + /* step over the memory occupied by vVals */ + excBuf = buf + dfor->item_cnt * sizeof(item_t); + excPossBuf = excBuf + dfor->exc_cnt * sizeof(item_t); + } + + res1 = vect_init(&vExcs, dfor->exc_cnt, (item_t *)excBuf); + res2 = vect_init(&vExcPoss, dfor->exc_cnt, (item_t *)excPossBuf); + + if (res1 != 0 || res2 != 0) + goto dfor_unpack_error; + + + for (size_t posExc = 0, j = 0; j < dfor->exc_cnt; j++) + { + item_t deltaPos; + res1 = vect_append(&vExcs, + bitpack_unpack(dfor->pack, &crExc, + dfor->exc_wid)); + /* Calculate the position of the exception from the delta of the + * position of the exception */ + deltaPos = bitpack_unpack(dfor->pack, &crPos, dfor->exc_pos_wid); + posExc += deltaPos; + res2 = vect_append(&vExcPoss, posExc); + if (res1 != 0 || res2 != 0) + goto dfor_unpack_error; + } + Assert(crExc == szDeltaPack + szExcPack); + Assert(crPos == + szDeltaPack + szExcPack + dfor->exc_pos_wid * dfor->exc_cnt); + } + + { /* Unpack deltas and calculate target values */ + item_t mDelta; + item_t mSum = 0; + size_t j = 0; /* index of an exception and its position in vectors */ + size_t crDelta = 0; + for (size_t i = 0; i < dfor->item_cnt; i++) + { + mDelta = bitpack_unpack(dfor->pack, &crDelta, dfor->delta_wid); + + if (isExcUsage == DFOR_EXC_USE && + j < vExcs.cnt && + i == vExcPoss.m[j]) + { + Assert(j < dfor->exc_cnt); + mDelta |= vExcs.m[j] << dfor->delta_wid; + j++; + } + mSum += mDelta; + vect_append(vVals, mSum); + } + Assert(crDelta == szDeltaPack); + res = 0; + } + +dfor_unpack_ret: + vect_clear(&vExcPoss); + vect_clear(&vExcs); + return res; +dfor_unpack_error: + vect_clear(vVals); + res = -1; + goto dfor_unpack_ret; +} + +void +dfor_clear_meta(dfor_meta_t *meta) +{ + if (meta == NULL) + return; + + if (meta->pack != NULL && !meta->outer_mem) + DFOR_FREE(meta->pack); + + memset(meta, 0, sizeof(dfor_meta_t)); +} + +dfor_stats_t +dfor_calc_stats(dfor_meta_t dfor) +{ + dfor_stats_t stat; + size_t nbytes; + stat.delta_pack_nbits = dfor.delta_wid * dfor.item_cnt; + stat.exc_pack_nbits = dfor.exc_wid * dfor.exc_cnt; + stat.exc_pos_pack_nbits = dfor.exc_pos_wid * dfor.exc_cnt; + + stat.nbits = stat.delta_pack_nbits + stat.exc_pack_nbits + stat.exc_pos_pack_nbits; + + /* If the division results in the remainder, we use an additional + * byte */ + nbytes = (stat.nbits + 7) / 8; + stat.ratio = (float)(sizeof(item_t) * dfor.item_cnt) / (float)nbytes; + + return stat; +} + +size_t dfor_calc_nbytes(dfor_meta_t dfor) +{ + dfor_stats_t stat; + stat = dfor_calc_stats(dfor); + return (stat.nbits + 7) / 8; +} + +#include "lib/dfor_templ_undef.h" diff --git a/src/backend/lib/dfor_u16.c b/src/backend/lib/dfor_u16.c new file mode 100644 index 00000000000..f7051f55925 --- /dev/null +++ b/src/backend/lib/dfor_u16.c @@ -0,0 +1,8 @@ +/* + * File: dfor_u16.c + */ + +/* clang-format off */ +#include "lib/dfor_u16_config.h" +#include "dfor_templ.c" +/* clang-format on */ diff --git a/src/backend/lib/meson.build b/src/backend/lib/meson.build index 0984bd0e3f6..7f6730efba1 100644 --- a/src/backend/lib/meson.build +++ b/src/backend/lib/meson.build @@ -2,6 +2,7 @@ dfor_sources = files( 'bitpack_u16.c', + 'dfor_u16.c', 'vect_u16.c' ) diff --git a/src/include/lib/dfor_templ.h b/src/include/lib/dfor_templ.h new file mode 100644 index 00000000000..b4c1d41c1d3 --- /dev/null +++ b/src/include/lib/dfor_templ.h @@ -0,0 +1,27 @@ +/* + * File: dfor_templ.h + */ +#include "dfor_templ_staple.h" + +extern int dfor_calc_deltas(size_t cnt, const item_t arr[], vect_t *vDeltas, + uniqsortvect_t *usvDeltaWidths, + vect_t *vWidthCounters); + +extern int dfor_calc_width(size_t cntDelta, + const uniqsortvect_t *usvDeltaWidths, + const vect_t *vWidthCounters, size_t *width, + size_t *cntExceptions); + +extern int dfor_pack(size_t cnt, const item_t arr[], excalg_t isExcUsage, + dfor_meta_t *dfor, size_t bufSize, uint8_t buf[]); + +extern int dfor_unpack(const dfor_meta_t *dfor, uniqsortvect_t *vVals, + size_t bufSize, uint8_t buf[]); + +extern void dfor_clear_meta(dfor_meta_t *dfor); + +extern dfor_stats_t dfor_calc_stats(dfor_meta_t dfor); + +extern size_t dfor_calc_nbytes(dfor_meta_t dfor); + +#include "dfor_templ_undef.h" diff --git a/src/include/lib/dfor_templ_staple.h b/src/include/lib/dfor_templ_staple.h new file mode 100644 index 00000000000..e93c40ac034 --- /dev/null +++ b/src/include/lib/dfor_templ_staple.h @@ -0,0 +1,125 @@ +/* + * File: dfor_staple_templ.h + */ + +#include "c.h" + +/****************************************************************************** + * NONREUSABLE CODE, allowed to be included only once + * + * This code should appear only once in a file (or in a chain of files) + * including this header with #include. So, we protect it from duplicates by + * means of macro + * + *****************************************************************************/ +#ifndef _DFOR_STAPLE_TEMPL_H_UNIQUE_CODE_ +#define _DFOR_STAPLE_TEMPL_H_UNIQUE_CODE_ + +typedef struct { + size_t item_cnt; + size_t delta_wid; + size_t exc_cnt; + size_t exc_wid; + size_t exc_pos_wid; + size_t nbytes; /* size of pack in bytes */ + uint8_t *pack; + bool outer_mem; +} dfor_meta_t; + +typedef struct { + size_t nbits; /* size of pack in bits used in fact */ + size_t delta_pack_nbits; /* in bits */ + size_t exc_pack_nbits; /* in bits */ + size_t exc_pos_pack_nbits; /* in bits */ + float ratio; /* compression ratio */ +} dfor_stats_t; + +typedef enum { + DFOR_EXC_DONT_USE = 0, + DFOR_EXC_USE = 1 +} excalg_t; + +#endif /* _DFOR_STAPLE_TEMPL_H_UNIQUE_CODE_ */ + +/****************************************************************************** + * End of Code, allowed to be included only once + *****************************************************************************/ + +/****************************************************************************** + * REUSABLE CODE + * + * This code can be reused with #include directive in a file if DFOR_MARKER is + * redefined. This allows creation of several types of vectors with different + * types of members. + * + *****************************************************************************/ + +#ifndef DFOR_ITEM_TYPE +#error "DFOR_ITEM_TYPE macro is indefined." +#endif + +#ifndef DFOR_MARKER +#error "DFOR_MARKER macro is indefined." +#endif + +#ifndef DFOR_MALLOC +#error "DFOR_MALLOC macro is indefined." +#endif + +#ifndef DFOR_FREE +#error "DFOR_FREE macro is indefined." +#endif + +#define MAKE_HEADER_NAME(v, m) CppAsString2(CppConcat2(v, m).h) + +/* + * Headers from vect and bitpack units + * + * Example: dfor_u16.c and dfor_u16.h need vect_u16.h and bitpack_u16.h + */ +#include MAKE_HEADER_NAME(lib/vect_, DFOR_MARKER) +#include MAKE_HEADER_NAME(lib/bitpack_, DFOR_MARKER) + +/* Types */ +#define item_t DFOR_ITEM_TYPE +#define vect_t CppConcatTriple2(vect_, DFOR_MARKER, _t) +#define uniqsortvect_t CppConcatTriple2(uniqsortvect_, DFOR_MARKER, _t) + +/* Functions */ +#define dfor_calc_deltas CppConcatTriple2(dfor_, DFOR_MARKER, _calc_deltas) +#define dfor_calc_width CppConcatTriple2(dfor_, DFOR_MARKER, _calc_width) +#define dfor_pack CppConcatTriple2(dfor_, DFOR_MARKER, _pack) +#define dfor_unpack CppConcatTriple2(dfor_, DFOR_MARKER, _unpack) +#define dfor_analyze CppConcatTriple2(dfor_, DFOR_MARKER, _analyze) +#define dfor_clear_meta CppConcatTriple2(dfor_, DFOR_MARKER, _clear_meta) +#define dfor_calc_stats CppConcatTriple2(dfor_, DFOR_MARKER, _calc_stats) +#define dfor_calc_nbytes CppConcatTriple2(dfor_, DFOR_MARKER, _calc_nbytes) + +/* Functions of the vect unit */ +#define vect_init CppConcatTriple2(vect_, DFOR_MARKER, _init) +#define vect_fill CppConcatTriple2(vect_, DFOR_MARKER, _fill) +#define vect_reserve CppConcatTriple2(vect_, DFOR_MARKER, _reserve) +#define vect_append CppConcatTriple2(vect_, DFOR_MARKER, _append) +#define vect_print CppConcatTriple2(vect_, DFOR_MARKER, _print) +#define vect_compare CppConcatTriple2(vect_, DFOR_MARKER, _compare) +#define vect_insert CppConcatTriple2(vect_, DFOR_MARKER, _insert) +#define vect_clear CppConcatTriple2(vect_, DFOR_MARKER, _clear) + +#define usv_insert CppConcatTriple2(usv_, DFOR_MARKER, _insert) +#define usv_search CppConcatTriple2(usv_, DFOR_MARKER, _search) + +/* Functions of the bitpack unit */ +#define width_from_val CppConcatTriple2(width_, DFOR_MARKER, _from_val) +#define width_to_mask CppConcatTriple2(width_, DFOR_MARKER, _to_mask) +#define bitpack_pack CppConcatTriple2(bitpack_, DFOR_MARKER, _pack) +#define bitpack_unpack CppConcatTriple2(bitpack_, DFOR_MARKER, _unpack) + +/****************************************************************************** + * End of Reusable Code + *****************************************************************************/ + +/* + * Don't forget to include this code in your file that uses this header + * #include "dfor_templ_undef.h" + * + */ diff --git a/src/include/lib/dfor_templ_undef.h b/src/include/lib/dfor_templ_undef.h new file mode 100644 index 00000000000..d60d3619308 --- /dev/null +++ b/src/include/lib/dfor_templ_undef.h @@ -0,0 +1,29 @@ +#undef item_t +#undef vect_t +#undef uniqsortvect_t + +#undef dfor_calc_deltas +#undef dfor_calc_width +#undef dfor_pack +#undef dfor_unpack +#undef dfor_analyze +#undef dfor_calc_stats +#undef dfor_calc_nbytes + +#undef vect_create +#undef vect_create_filled +#undef vect_reserve +#undef vect_append +#undef vect_destroy +#undef vect_print +#undef vect_compare +#undef vect_insert +#undef vect_clear + +#undef usv_insert +#undef usv_search + +#undef width_from_val +#undef width_to_mask +#undef bitpack_pack +#undef bitpack_unpack diff --git a/src/include/lib/dfor_u16.h b/src/include/lib/dfor_u16.h new file mode 100644 index 00000000000..716c99dbc55 --- /dev/null +++ b/src/include/lib/dfor_u16.h @@ -0,0 +1,13 @@ +/* + * File: dfor_u16.h + */ + +#ifndef _DFOR_U16_H_ +#define _DFOR_U16_H_ + +/* clang-format off */ +#include "dfor_u16_config.h" +#include "dfor_templ.h" +/* clang-format on */ + +#endif /* _DFOR_U16_H_ */ diff --git a/src/include/lib/dfor_u16_config.h b/src/include/lib/dfor_u16_config.h new file mode 100644 index 00000000000..751937ac513 --- /dev/null +++ b/src/include/lib/dfor_u16_config.h @@ -0,0 +1,4 @@ +#define DFOR_ITEM_TYPE uint16_t +#define DFOR_MARKER u16 +#define DFOR_MALLOC malloc +#define DFOR_FREE free diff --git a/src/test/dfor/.gitignore b/src/test/dfor/.gitignore index 0d77a51216b..447e95c0c09 100644 --- a/src/test/dfor/.gitignore +++ b/src/test/dfor/.gitignore @@ -1,3 +1,4 @@ test_bitpack_u16 +test_dfor_u16 test_uniqsortvect_u16 test_vect_u16 diff --git a/src/test/dfor/Makefile b/src/test/dfor/Makefile index a15d5eaf736..1af529c7378 100644 --- a/src/test/dfor/Makefile +++ b/src/test/dfor/Makefile @@ -40,7 +40,8 @@ LIBTAP_OBJS = $(top_builddir)/src/test/libtap/tap.o TESTS= test_vect_u16 \ test_uniqsortvect_u16 \ - test_bitpack_u16 + test_bitpack_u16 \ + test_dfor_u16 $(TESTS:%=%.o): CPPFLAGS += -I$(top_builddir)/src/test -DFRONTEND diff --git a/src/test/dfor/meson.build b/src/test/dfor/meson.build index ce762c52430..4a760ab68fa 100644 --- a/src/test/dfor/meson.build +++ b/src/test/dfor/meson.build @@ -8,6 +8,7 @@ dfor_dir = join_paths(meson.project_source_root(), 'src/backend/lib') dfor_sources = files( join_paths(dfor_dir, 'vect_u16.c'), join_paths(dfor_dir, 'bitpack_u16.c'), + join_paths(dfor_dir, 'dfor_u16.c'), ) dfor_test_lib = static_library( @@ -36,6 +37,7 @@ test_names = [ 'test_vect_u16', 'test_uniqsortvect_u16', 'test_bitpack_u16', + 'test_dfor_u16', ] foreach t : test_names diff --git a/src/test/dfor/test_dfor_u16.c b/src/test/dfor/test_dfor_u16.c new file mode 100644 index 00000000000..322b714ba38 --- /dev/null +++ b/src/test/dfor/test_dfor_u16.c @@ -0,0 +1,371 @@ +/* + * test_dfor.c + */ + +#include "lib/bitpack_u16.h" +#include "lib/dfor_u16.h" +#include "lib/vect_u16.h" +#include "libtap/tap.h" +#include "test.h" + +void test_delta_calculation(size_t cnt, uint16_t inArr[], size_t cntDelta, + uint16_t marDeltasExpected[], size_t cntWidth, + uint16_t marWidthsExpected[], size_t cntStat, + uint16_t marWidthsStatExpected[]); + +void test_calc_exceptions(size_t numDeltas, size_t numWidths, + uint16_t marWidths[], size_t numCounts, + uint16_t marCounts[], size_t szAwaitedWidth, + size_t cntAwaitedExcCount); + +void test_dfor(size_t cnt, uint16_t arr[], excalg_t isExcUsage, + size_t widDeltaAwaited, size_t cntExcCntAwaited, + size_t widExcWidAwaited, size_t widExcPosWidAwaited, + size_t cntBitsCountAwaited, size_t cntByteCountAwaited, + float flMinRatioAwaited, uint8_t u8arPackAwaited[]); + +void +test_delta_calculation(size_t cnt, uint16_t inArr[], size_t cntDelta, + uint16_t marDeltasExpected[], size_t cntWidth, + uint16_t marWidthsExpected[], size_t cntStat, + uint16_t marWidthsStatExpected[]) +{ + vect_u16_t vDeltas; + vect_u16_t vWidthCounters; + uniqsortvect_u16_t usvDeltaWidths; + vect_u16_t awaited; + int res; + + printf("------------------------------------------------\n"); + printf("Test\n"); + printf("------------------------------------------------\n"); + + printf(" inArr:"); + for (size_t i = 0; i < cnt; i++) + printf(" %u", (uint32_t)inArr[i]); + printf("\n"); + + vect_u16_init(&vDeltas, 0, NULL); + vect_u16_init(&usvDeltaWidths, 0, NULL); + vect_u16_init(&vWidthCounters, 0, NULL); + + /* Tested function */ + res = dfor_u16_calc_deltas(cnt, inArr, &vDeltas, &usvDeltaWidths, + &vWidthCounters); + cmp_ok(res, "==", 0); + + printf(" Delta expected:"); + for (size_t i = 0; i < cntDelta; i++) + printf(" %u", (uint32_t)marDeltasExpected[i]); + printf("\n"); + + printf(" Delta fact: "); + vect_u16_print(&vDeltas); + + cmp_ok(vDeltas.cnt, "==", cnt, "The Delta count is OK."); + vect_u16_init(&awaited, 0, NULL); + vect_u16_fill(&awaited, cnt, marDeltasExpected); + cmp_ok(vect_u16_compare(&vDeltas, &awaited), "==", 0, + "All deltas are calculated properly"); + vect_u16_clear(&awaited); + + printf(" Width expected:"); + for (size_t i = 0; i < cntWidth; i++) + printf(" %u", (uint32_t)marWidthsExpected[i]); + printf("\n"); + + printf(" Width fact: "); + vect_u16_print(&usvDeltaWidths); + + cmp_ok(usvDeltaWidths.cnt, "==", cntWidth, "The Width count is OK."); + + /* don't really need initialisation after vect_clean having been done + * above*/ + /* vect_u16_init(&awaited, 0, NULL); */ + + vect_u16_fill(&awaited, cntWidth, marWidthsExpected); + cmp_ok(vect_u16_compare(&usvDeltaWidths, &awaited), "==", 0, + "All delta widths is OK."); + vect_u16_clear(&awaited); + + printf(" Statistics expected:"); + for (size_t i = 0; i < cntStat; i++) + printf(" %u", (uint32_t)marWidthsStatExpected[i]); + printf("\n"); + + printf(" Statistics fact: "); + vect_u16_print(&vWidthCounters); + + cmp_ok( + usvDeltaWidths.cnt, "==", vWidthCounters.cnt, + "The count of statistics of widths is equal to the count of widths."); + + /* don't really need initialisation after vect_clean having been done + * above*/ + vect_u16_fill(&awaited, cntStat, marWidthsStatExpected); + cmp_ok(vect_u16_compare(&vWidthCounters, &awaited), "==", 0, + "Width statistics is OK."); + vect_u16_clear(&awaited); + + vect_u16_clear(&vDeltas); + vect_u16_clear(&usvDeltaWidths); + vect_u16_clear(&vWidthCounters); +} + +void +test_calc_exceptions(size_t numDeltas, size_t numWidths, uint16_t marWidths[], + size_t numCounts, uint16_t marCounts[], + size_t szAwaitedWidth, size_t cntAwaitedExcCount) +{ + int res; + vect_u16_t vWidthCounters; + uniqsortvect_u16_t usvDeltaWidths; + size_t width, cntExceptions; + + vect_u16_init(&usvDeltaWidths, 0, NULL); + vect_u16_fill(&usvDeltaWidths, numWidths, marWidths); + + vect_u16_init(&vWidthCounters, 0, NULL); + vect_u16_fill(&vWidthCounters, numCounts, marCounts); + + res = dfor_u16_calc_width(numDeltas, &usvDeltaWidths, &vWidthCounters, + &width, &cntExceptions); + cmp_ok(res, "==", 0); + cmp_ok(width, "==", szAwaitedWidth, "Width is OK."); + cmp_ok(cntExceptions, "==", cntAwaitedExcCount, "Exceptions num is OK"); + + vect_u16_clear(&usvDeltaWidths); + vect_u16_clear(&vWidthCounters); +} + +void +test_dfor(size_t cnt, uint16_t arr[], excalg_t isExcUsage, + size_t widDeltaWidthAwaited, size_t cntExcCntAwaited, + size_t widExcWidAwaited, size_t widExcPosWidAwaited, + size_t cntBitsCountAwaited, size_t cntByteCountAwaited, + float flMinRatioAwaited, uint8_t u8arPackAwaited[]) +{ + int res; + dfor_meta_t dfor; + dfor_stats_t stats; + uniqsortvect_u16_t extracted; + + res = dfor_u16_pack(cnt, arr, isExcUsage, &dfor, 0, NULL); + cmp_ok(res, "==", 0, "dfor_pack func has processed OK."); + cmp_ok(dfor.item_cnt, "==", cnt, "Count of deltas is OK."); + cmp_ok(dfor.delta_wid, "==", widDeltaWidthAwaited, "Delta width is OK."); + cmp_ok(dfor.exc_cnt, "==", cntExcCntAwaited, "Exception count is OK."); + cmp_ok(dfor.exc_wid, "==", widExcWidAwaited, "Exception width is OK."); + cmp_ok(dfor.exc_pos_wid, "==", widExcPosWidAwaited, + "Exception position width is OK."); + ok(dfor.pack != NULL, "Pack is created (not NULL)."); + + stats = dfor_u16_calc_stats(dfor); + cmp_ok(stats.nbits, "==", cntBitsCountAwaited, "Bits count is OK."); + cmp_ok(dfor.nbytes, "==", cntByteCountAwaited, "Bytes count is OK."); + ok(stats.ratio > flMinRatioAwaited, "Compression ratio is OK."); + + if (u8arPackAwaited != NULL) + ok(0 == memcmp(dfor.pack, u8arPackAwaited, cntByteCountAwaited), + "Pack content is OK."); + else + ok(0 == 0, "Pack content check is skipped."); + + test_print_u16_array(cnt, (uint16_t *)arr, "\n\nOriginal integer array"); + test_print_u8_array(dfor.nbytes, dfor.pack, "Compressed integer array"); + printf("Compression ratio:%f\n\n", stats.ratio); + + vect_u16_init(&extracted, 0, NULL); + + dfor_u16_unpack(&dfor, &extracted, 0, NULL); + cmp_ok(extracted.cnt, "==", cnt, "Extracted count is OK"); + cmp_ok(0, "==", memcmp(arr, extracted.m, cnt), + "Extracted array is equal to original"); + + free(dfor.pack); + vect_u16_clear(&extracted); +} + +int +main(void) +{ + plan(130); + printf("========================================\n"); + printf("Test DELTA CALCULATION\n"); + { + test_delta_calculation( + 10, (uint16_t[]) { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, /* inArr */ + 10, + (uint16_t[]) { 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1 }, /* marDeltasExpected*/ + 1, (uint16_t[]) { 1 }, /* marWidthsExpected */ + 1, (uint16_t[]) { 10 } /* marWidthsStatExpected */); + + test_delta_calculation( + 10, (uint16_t[]) { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19 }, /* inArr */ + 10, + (uint16_t[]) { 1, 2, 2, 2, 2, 2, 2, 2, 2, + 2 }, /* marDeltasExpected*/ + 2, (uint16_t[]) { 1, 2 }, /* marWidthsExpected */ + 2, (uint16_t[]) { 1, 9 } /* marWidthsStatExpected */); + + test_delta_calculation( + 14, + (uint16_t[]) { 100, 200, 300, 400, 401, 402, 403, 404, 406, 408, + 410, 412, 414, 416 }, /* inArr */ + 14, + (uint16_t[]) { 100, 100, 100, 100, 1, 1, 1, 1, 2, 2, 2, 2, 2, + 2 }, /* marDeltasExpected*/ + 3, (uint16_t[]) { 1, 2, 7 }, /* marWidthsExpected */ + 3, (uint16_t[]) { 4, 6, 4 } /* marWidthsStatExpected */); + + test_delta_calculation(1, (uint16_t[]) { 123 }, /* inArr */ + 1, (uint16_t[]) { 123 }, /* marDeltasExpected*/ + 1, (uint16_t[]) { 7 }, /* marWidthsExpected */ + 1, + (uint16_t[]) { 1 } /* marWidthsStatExpected */); + + test_delta_calculation(0, NULL, /* inArr */ + 0, NULL, /* marDeltasExpected*/ + 0, NULL, /* marWidthsExpected */ + 0, NULL /* marWidthsStatExpected */); + } + printf("Test DELTA CALCULATION PASSED\n"); + printf("========================================\n\n"); + + printf("========================================\n"); + printf("Test EXCEPTIONS CALCULATION\n"); + { + int res; + vect_u16_t vWidthCounters; + uniqsortvect_u16_t usvDeltaWidths; + size_t width, cntExceptions; + + vect_u16_init(&usvDeltaWidths, 0, NULL); + vect_u16_fill(&usvDeltaWidths, 3, (uint16_t[]) { 1, 2, 7 }); + + vect_u16_init(&vWidthCounters, 0, NULL); + vect_u16_fill(&vWidthCounters, 3, (uint16_t[]) { 4, 6, 4 }); // 4+6+4=14 + + res = dfor_u16_calc_width(14, &usvDeltaWidths, &vWidthCounters, &width, + &cntExceptions); + cmp_ok(res, "==", 0); + cmp_ok(width, "==", 7, "Widths={1,2,7}, Counters={4,6,4} => width=7"); + cmp_ok(cntExceptions, "==", 0, + "Widths={1,2,7}, Counters={4,6,4} => excptions_num=0"); + } + + test_calc_exceptions(14, 3, (uint16_t[]) { 1, 2, 7 }, /* widths of deltas*/ + 3, (uint16_t[]) { 4, 6, 4 }, /* statistics */ + 7, /* width of short deltas */ + 0); /* number of exceptions*/ + + test_calc_exceptions(14, 3, (uint16_t[]) { 1, 2, 7 }, /* widths */ + 3, (uint16_t[]) { 6, 7, 1 }, /* stat */ + 2, 1); /* short deltas width, exceptions count*/ + + test_calc_exceptions(40, 3, (uint16_t[]) { 5, 6, 12 }, /* widths */ + 3, (uint16_t[]) { 36, 2, 2 }, /* stat */ + 5, 4); /* short deltas width, exceptions count*/ + + test_calc_exceptions(40, 4, (uint16_t[]) { 5, 6, 7, 12 }, 4, + (uint16_t[]) { 36, 1, 1, 2 }, 5, 4); + + test_calc_exceptions(40, 4, (uint16_t[]) { 5, 6, 7, 12 }, 4, + (uint16_t[]) { 35, 1, 2, 2 }, 6, 4); + + test_calc_exceptions(40, 4, (uint16_t[]) { 5, 6, 7, 12 }, 4, + (uint16_t[]) { 34, 1, 2, 3 }, 7, 3); + + test_calc_exceptions(40, 4, (uint16_t[]) { 5, 6, 7, 12 }, 4, + (uint16_t[]) { 1, 34, 2, 3 }, 7, 3); + + test_calc_exceptions(40, 4, (uint16_t[]) { 5, 6, 7, 12 }, 4, + (uint16_t[]) { 1, 33, 2, 4 }, 7, 4); + + test_calc_exceptions(40, 4, (uint16_t[]) { 5, 6, 7, 12 }, 4, + (uint16_t[]) { 1, 32, 2, 5 }, 12, 0); + + printf("Test EXCEPTIONS CALCULATION PASSED\n"); + printf("========================================\n\n"); + + printf("========================================\n"); + printf("Test DELTA FRAME OF REFERENCES PACKING\n"); + + test_dfor(16, + (uint16_t[]) { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15 }, /* cnt, array */ + DFOR_EXC_DONT_USE, /* flag on use of exceptions */ + 1, /* awaited width of short deltas */ + 0, /* awaited count of exceptions */ + 0, /* awaited exception width*/ + 0, /* awaited exception position width*/ + 16, /* awaited bits count */ + 2, /* cntByteCountAwaited */ + 15.99, /* awaited ratio min value (ratio >= 15.99) */ + (uint8_t[]) { 0xFE, 0xFF }); + + test_dfor(16, + (uint16_t[]) { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15 }, /* cnt, array */ + DFOR_EXC_USE, /* flag on use of exceptions */ + 1, /* awaited width of short deltas */ + 0, /* awaited count of exceptions */ + 0, /* awaited exception width*/ + 0, /* awaited exception position width*/ + 16, /* awaited bits count */ + 2, /* cntByteCountAwaited */ + 15.99, /* awaited ratio min value (ratio >= 15.99) */ + (uint8_t[]) { 0xFE, 0xFF }); + + test_dfor(16, + (uint16_t[]) { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 1023 }, /* cnt, array */ + DFOR_EXC_DONT_USE, /* flag on use of exceptions */ + 10, /* awaited width of short deltas */ + 0, /* awaited count of exceptions */ + 0, /* awaited exception width*/ + 0, /* awaited exception position width*/ + 10 * 16, /* awaited bits count */ + 20, /* awaited bytes count */ + 1.5, /* awaited ratio min value */ + (uint8_t[]) { 0x00, 0x04, 0x10, 0x40, 0x00, 0x01, 0x04, + 0x10, 0x40, 0x00, 0x01, 0x04, 0x10, 0x40, + 0x00, 0x01, 0x04, 0x10, 0x40, 0xFC }); + + test_dfor(16, + (uint16_t[]) { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 1023 }, /* cnt, array */ + DFOR_EXC_USE, /* flag on use of exceptions */ + 1, /* awaited width of short deltas */ + 1, /* awaited count of exceptions */ + 9, /* awaited exception width*/ + 4, /* awaited exception position width*/ + 16 * 1 + 9 + 4, /* awaited bits count */ + 4, /* awaited bytes count */ + 7.99, /* awaited ratio min value */ + (uint8_t[]) { 0xFE, 0xFF, 0xF8, 0x1F }); + + test_dfor(30, /* cnt */ + (uint16_t[]) { 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, + /* delta=2, pos=10, deltapos=10 */ 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, + /* delta=3, pos=25, deltapos=15 */ 28, 29, 30, 31, + /* delta=3, pos=29, deltapos=4 */ 34 }, /* array */ + DFOR_EXC_USE, /* flag on use of exceptions */ + 1, /* awaited width of short deltas */ + 3, /* awaited count of exceptions */ + 1, /* awaited exception width*/ + 4, /* awaited exception position width*/ + 30 * 1 + 3 * 1 + 3 * 4, /* awaited bits count */ + 6, /* awaited bytes count */ + 9.99, /* awaited ratio min value */ + (uint8_t[]) { 0xFE, 0xFB, 0xFF, 0xFF, 0xF5, 0x09 }); + + printf("Test DELTA FRAME OF REFERENCES PACKING PASSED\n"); + printf("========================================\n\n"); + + done_testing(); +} -- 2.53.0