diff --git a/src/backend/access/common/toast_dictionary.c b/src/backend/access/common/toast_dictionary.c index dbcbced460..a4f61c63e5 100644 --- a/src/backend/access/common/toast_dictionary.c +++ b/src/backend/access/common/toast_dictionary.c @@ -542,38 +542,50 @@ zstd_dict_decompress_datum_slice(const varlena *value, int32 slicelength) } /* - * Compressed size of one value against raw dictionary bytes, for the held-out - * test of dictionary evolution. Builds a one-shot CDict from the given bytes - * (not the per-backend digested cache, which keys on OID and a candidate has - * no OID yet) and returns the compressed byte count without storing anything. - * Returns 0 on any zstd failure so the caller can treat it as "no estimate." + * Total compressed size of the held-out values against raw dictionary bytes, + * for the held-out test of dictionary evolution. Digests the bytes into a + * one-shot CDict once for all the values (not the per-backend digested cache, + * which keys on OID and a candidate has no OID yet) and returns the total + * compressed byte count without storing anything. Returns 0 on any zstd + * failure so the caller can treat it as "no estimate." */ static size_t -zstd_dict_compressed_size(const char *dict, size_t dictlen, - const char *val, size_t len) +zstd_dict_compressed_total(const char *dict, size_t dictlen, + const char *vals, const size_t *sizes, int nvals) { #ifndef USE_ZSTD return 0; #else ZSTD_CDict *cd; - size_t bound = ZSTD_compressBound(len); - char *out = palloc(bound); - size_t clen; + size_t total = 0; + size_t maxlen = 0; + char *out; + for (int i = 0; i < nvals; i++) + maxlen = Max(maxlen, sizes[i]); if (dict_cctx == NULL && (dict_cctx = ZSTD_createCCtx()) == NULL) elog(ERROR, "could not create zstd compression context"); cd = ZSTD_createCDict(dict, dictlen, ZSTD_CLEVEL_DEFAULT); if (cd == NULL) - { - pfree(out); return 0; + out = palloc(ZSTD_compressBound(maxlen)); + for (int i = 0; i < nvals; i++) + { + size_t clen = ZSTD_compress_usingCDict(dict_cctx, out, + ZSTD_compressBound(maxlen), + vals, sizes[i], cd); + + if (ZSTD_isError(clen)) + { + total = 0; + break; + } + total += clen; + vals += sizes[i]; } - clen = ZSTD_compress_usingCDict(dict_cctx, out, bound, val, len, cd); ZSTD_freeCDict(cd); pfree(out); - if (ZSTD_isError(clen)) - return 0; - return clen; + return total; #endif } @@ -729,36 +741,44 @@ lz4_dict_decompress_datum_slice(const varlena *value, int32 slicelength) } /* - * Compressed size of one value against raw dictionary bytes, for the held-out - * test of dictionary evolution. Loads the bytes into a fresh lz4 stream and - * returns the compressed byte count without storing anything. Returns 0 on - * any lz4 failure. + * Total compressed size of the held-out values against raw dictionary bytes, + * for the held-out test of dictionary evolution. Loads the bytes into a fresh + * lz4 stream for each value, as compression does, and returns the total + * compressed byte count without storing anything. Returns 0 on any lz4 + * failure. */ static size_t -lz4_dict_compressed_size(const char *dict, size_t dictlen, - const char *val, size_t len) +lz4_dict_compressed_total(const char *dict, size_t dictlen, + const char *vals, const size_t *sizes, int nvals) { #ifndef USE_LZ4 return 0; #else - LZ4_stream_t *stream; - int32 bound = LZ4_compressBound(len); - char *out; - int32 clen; + size_t total = 0; - if (bound <= 0) - return 0; - stream = LZ4_createStream(); - if (stream == NULL) - return 0; - LZ4_loadDict(stream, dict, dictlen); - out = palloc(bound); - clen = LZ4_compress_fast_continue(stream, val, out, len, bound, 1); - LZ4_freeStream(stream); - pfree(out); - if (clen <= 0) - return 0; - return clen; + for (int i = 0; i < nvals; i++) + { + LZ4_stream_t *stream; + int32 bound = LZ4_compressBound(sizes[i]); + char *out; + int32 clen; + + if (bound <= 0) + return 0; + stream = LZ4_createStream(); + if (stream == NULL) + return 0; + LZ4_loadDict(stream, dict, dictlen); + out = palloc(bound); + clen = LZ4_compress_fast_continue(stream, vals, out, sizes[i], bound, 1); + LZ4_freeStream(stream); + pfree(out); + if (clen <= 0) + return 0; + total += clen; + vals += sizes[i]; + } + return total; #endif } @@ -770,7 +790,7 @@ static const DictCompressRoutine lz4_dict_routine = { .decompress = lz4_dict_decompress_datum, .decompress_slice = lz4_dict_decompress_datum_slice, .train = dict_train_zdict, - .compressed_size = lz4_dict_compressed_size, + .compressed_total = lz4_dict_compressed_total, }; /* @@ -782,7 +802,7 @@ static const DictCompressRoutine zstd_dict_routine = { .decompress = zstd_dict_decompress_datum, .decompress_slice = zstd_dict_decompress_datum_slice, .train = dict_train_zdict, - .compressed_size = zstd_dict_compressed_size, + .compressed_total = zstd_dict_compressed_total, }; /* @@ -1335,29 +1355,16 @@ toast_dictionary_maybe_evolve(Relation onerel, AttrNumber attnum, adopt = false; /* nothing to judge by; keep current */ else { - size_t curtotal = 0; - size_t candtotal = 0; - const char *p = held.data; - bool ok = true; - - for (i = 0; i < nheld; i++) - { - size_t c = routine->compressed_size(VARDATA_ANY(curbytes), - VARSIZE_ANY_EXHDR(curbytes), - p, heldsizes[i]); - size_t n = routine->compressed_size(candbuf, candlen, - p, heldsizes[i]); - - if (c == 0 || n == 0) - { - ok = false; - break; - } - curtotal += c; - candtotal += n; - p += heldsizes[i]; - } - adopt = ok && + /* one call per dictionary: each is digested once, not per value */ + size_t curtotal = routine->compressed_total(VARDATA_ANY(curbytes), + VARSIZE_ANY_EXHDR(curbytes), + held.data, heldsizes, + nheld); + size_t candtotal = routine->compressed_total(candbuf, candlen, + held.data, heldsizes, + nheld); + + adopt = curtotal != 0 && candtotal != 0 && (double) candtotal <= (double) curtotal * (1.0 - dictionary_evolution_min_gain); diff --git a/src/include/access/toast_dictionary.h b/src/include/access/toast_dictionary.h index 169f801653..ce368d785e 100644 --- a/src/include/access/toast_dictionary.h +++ b/src/include/access/toast_dictionary.h @@ -40,11 +40,13 @@ * evolution (see toast_dictionary_maybe_evolve()). train() builds a * dictionary of at most max_dict_bytes from nsamples concatenated samples * (sizes[] gives each one's length), writing it into dictbuf and returning - * its length, or 0 on failure (a decline, not an error). compressed_size() - * compresses one value against raw dictionary bytes and returns the - * compressed byte count without storing anything, so the evolution policy can - * weigh a candidate dictionary against the current one. Both are codec - * contract points only: the policy never interprets the dictionary bytes. + * its length, or 0 on failure (a decline, not an error). compressed_total() + * compresses each of nvals concatenated values (sizes[] gives each one's + * length) against raw dictionary bytes and returns the total compressed byte + * count without storing anything, or 0 if any value fails, so the evolution + * policy can weigh a candidate dictionary against the current one with one + * call per dictionary. Both are codec contract points only: the policy never + * interprets the dictionary bytes. */ typedef struct DictCompressRoutine { @@ -53,8 +55,9 @@ typedef struct DictCompressRoutine varlena *(*decompress_slice) (const varlena *value, int32 slicelength); size_t (*train) (const char *samples, const size_t *sizes, int nsamples, size_t max_dict_bytes, void *dictbuf); - size_t (*compressed_size) (const char *dict, size_t dictlen, - const char *val, size_t len); + size_t (*compressed_total) (const char *dict, size_t dictlen, + const char *vals, const size_t *sizes, + int nvals); } DictCompressRoutine; /* Current dictionary of rel.attnum, or InvalidOid if none */