From 3229f3c90519f9ad441821c2a819429ad34f9011 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Tue, 31 Dec 2024 09:19:24 +1300 Subject: [PATCH v5 2/2] Precalculate CompactAttribute's attcacheoff This allows code to be removed from the tuple deform routines which shrinks down the code a little, which can make it run more quickly. This also makes a dedicated deformer loop to deform the portion of the tuple which has a known offset, which makes deforming much faster when a leading set of the table's columns are non-NULL values and fixed-width types. --- src/backend/access/common/heaptuple.c | 334 +++++++++------------- src/backend/access/common/indextuple.c | 367 ++++++++++--------------- src/backend/access/common/tupdesc.c | 37 +++ src/backend/access/spgist/spgutils.c | 3 - src/backend/executor/execTuples.c | 280 ++++++++----------- src/backend/jit/llvm/llvmjit_deform.c | 6 - src/backend/utils/cache/relcache.c | 12 - src/include/access/htup_details.h | 19 +- src/include/access/itup.h | 20 +- src/include/access/tupdesc.h | 10 +- src/include/access/tupmacs.h | 65 +++++ src/include/executor/tuptable.h | 9 +- 12 files changed, 522 insertions(+), 640 deletions(-) diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index 11bec20e82e..42cce3dcdfe 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -497,20 +497,8 @@ heap_attisnull(HeapTuple tup, int attnum, TupleDesc tupleDesc) /* ---------------- * nocachegetattr * - * This only gets called from fastgetattr(), in cases where we - * can't use a cacheoffset and the value is not null. - * - * This caches attribute offsets in the attribute descriptor. - * - * An alternative way to speed things up would be to cache offsets - * with the tuple, but that seems more difficult unless you take - * the storage hit of actually putting those offsets into the - * tuple you send to disk. Yuck. - * - * This scheme will be slightly slower than that, but should - * perform well for queries which hit large #'s of tuples. After - * you cache the offsets once, examining all the other tuples using - * the same attribute descriptor will go much quicker. -cim 5/4/91 + * This only gets called from fastgetattr(), in cases where the + * attcacheoff is not set. * * NOTE: if you need to change this code, see also heap_deform_tuple. * Also see nocache_index_getattr, which is the same code for index @@ -522,194 +510,104 @@ nocachegetattr(HeapTuple tup, int attnum, TupleDesc tupleDesc) { + CompactAttribute *cattr; HeapTupleHeader td = tup->t_data; char *tp; /* ptr to data part of tuple */ bits8 *bp = td->t_bits; /* ptr to null bitmap in tuple */ - bool slow = false; /* do we have to walk attrs? */ int off; /* current offset within data */ + int startAttr; + int firstnullattr; + bool hasnulls = HeapTupleHasNulls(tup); - /* ---------------- - * Three cases: - * - * 1: No nulls and no variable-width attributes. - * 2: Has a null or a var-width AFTER att. - * 3: Has nulls or var-widths BEFORE att. - * ---------------- - */ + /* Did someone forget to call TupleDescFinalize()? */ + Assert(tupleDesc->firstNonCachedOffAttr >= 0); + /* + * If there are no NULLs before the required attnum, then we can start at + * the highest attribute with a known offset, or the first attribute if + * none have a cached offset. If the tuple has no variable width types, + * then we can use a slightly cheaper method of offset calculation, as we + * just need to add the attlen to the aligned offset when skipping over + * columns. When the tuple contains variable-width types, we must use + * att_addlength_pointer(), which does a bit more branching and is + * slightly less efficient. + */ attnum--; - if (!HeapTupleNoNulls(tup)) - { - /* - * there's a null somewhere in the tuple - * - * check to see if any preceding bits are null... - */ - int byte = attnum >> 3; - int finalbit = attnum & 0x07; - - /* check for nulls "before" final bit of last byte */ - if ((~bp[byte]) & ((1 << finalbit) - 1)) - slow = true; - else - { - /* check for nulls in any "earlier" bytes */ - int i; + if (hasnulls) + firstnullattr = first_null_attr(bp, attnum); + else + firstnullattr = attnum; - for (i = 0; i < byte; i++) - { - if (bp[i] != 0xFF) - { - slow = true; - break; - } - } - } + if (tupleDesc->firstNonCachedOffAttr > 0) + { + startAttr = Min(tupleDesc->firstNonCachedOffAttr - 1, firstnullattr); + off = TupleDescCompactAttr(tupleDesc, startAttr)->attcacheoff; + } + else + { + startAttr = 0; + off = 0; } tp = (char *) td + td->t_hoff; - if (!slow) + if (hasnulls) { - CompactAttribute *att; + for (int i = startAttr; i < attnum; i++) + { + CompactAttribute *att; - /* - * If we get here, there are no nulls up to and including the target - * attribute. If we have a cached offset, we can use it. - */ - att = TupleDescCompactAttr(tupleDesc, attnum); - if (att->attcacheoff >= 0) - return fetchatt(att, tp + att->attcacheoff); + if (att_isnull(i, bp)) + continue; - /* - * Otherwise, check for non-fixed-length attrs up to and including - * target. If there aren't any, it's safe to cheaply initialize the - * cached offsets for these attrs. - */ - if (HeapTupleHasVarWidth(tup)) - { - int j; + att = TupleDescCompactAttr(tupleDesc, i); - for (j = 0; j <= attnum; j++) - { - if (TupleDescCompactAttr(tupleDesc, j)->attlen <= 0) - { - slow = true; - break; - } - } + off = att_pointer_alignby(off, + att->attalignby, + att->attlen, + tp + off); + off = att_addlength_pointer(off, att->attlen, tp + off); } + cattr = TupleDescCompactAttr(tupleDesc, attnum); + off = att_pointer_alignby(off, + cattr->attalignby, + cattr->attlen, + tp + off); } - - if (!slow) + else if (!HeapTupleHasVarWidth(tup)) { - int natts = tupleDesc->natts; - int j = 1; - - /* - * If we get here, we have a tuple with no nulls or var-widths up to - * and including the target attribute, so we can use the cached offset - * ... only we don't have it yet, or we'd not have got here. Since - * it's cheap to compute offsets for fixed-width columns, we take the - * opportunity to initialize the cached offsets for *all* the leading - * fixed-width columns, in hope of avoiding future visits to this - * routine. - */ - TupleDescCompactAttr(tupleDesc, 0)->attcacheoff = 0; - - /* we might have set some offsets in the slow path previously */ - while (j < natts && TupleDescCompactAttr(tupleDesc, j)->attcacheoff > 0) - j++; - - off = TupleDescCompactAttr(tupleDesc, j - 1)->attcacheoff + - TupleDescCompactAttr(tupleDesc, j - 1)->attlen; - - for (; j < natts; j++) + for (int i = startAttr; i < attnum; i++) { - CompactAttribute *att = TupleDescCompactAttr(tupleDesc, j); - - if (att->attlen <= 0) - break; + CompactAttribute *att = TupleDescCompactAttr(tupleDesc, i); off = att_nominal_alignby(off, att->attalignby); - - att->attcacheoff = off; - off += att->attlen; } - - Assert(j > attnum); - - off = TupleDescCompactAttr(tupleDesc, attnum)->attcacheoff; + cattr = TupleDescCompactAttr(tupleDesc, attnum); + off = att_nominal_alignby(off, cattr->attalignby); } else { - bool usecache = true; - int i; - - /* - * Now we know that we have to walk the tuple CAREFULLY. But we still - * might be able to cache some offsets for next time. - * - * Note - This loop is a little tricky. For each non-null attribute, - * we have to first account for alignment padding before the attr, - * then advance over the attr based on its length. Nulls have no - * storage and no alignment padding either. We can use/set - * attcacheoff until we reach either a null or a var-width attribute. - */ - off = 0; - for (i = 0;; i++) /* loop exit is at "break" */ + for (int i = startAttr; i < attnum; i++) { CompactAttribute *att = TupleDescCompactAttr(tupleDesc, i); - if (HeapTupleHasNulls(tup) && att_isnull(i, bp)) - { - usecache = false; - continue; /* this cannot be the target att */ - } - - /* If we know the next offset, we can skip the rest */ - if (usecache && att->attcacheoff >= 0) - off = att->attcacheoff; - else if (att->attlen == -1) - { - /* - * We can only cache the offset for a varlena attribute if the - * offset is already suitably aligned, so that there would be - * no pad bytes in any case: then the offset will be valid for - * either an aligned or unaligned value. - */ - if (usecache && - off == att_nominal_alignby(off, att->attalignby)) - att->attcacheoff = off; - else - { - off = att_pointer_alignby(off, att->attalignby, -1, - tp + off); - usecache = false; - } - } - else - { - /* not varlena, so safe to use att_nominal_alignby */ - off = att_nominal_alignby(off, att->attalignby); - - if (usecache) - att->attcacheoff = off; - } - - if (i == attnum) - break; - + off = att_pointer_alignby(off, + att->attalignby, + att->attlen, + tp + off); off = att_addlength_pointer(off, att->attlen, tp + off); - if (usecache && att->attlen <= 0) - usecache = false; } + cattr = TupleDescCompactAttr(tupleDesc, attnum); + off = att_pointer_alignby(off, + cattr->attalignby, + cattr->attlen, + tp + off); } - return fetchatt(TupleDescCompactAttr(tupleDesc, attnum), tp + off); + return fetchatt(cattr, tp + off); } /* ---------------- @@ -1347,6 +1245,7 @@ heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, bool *isnull) { HeapTupleHeader tup = tuple->t_data; + CompactAttribute *cattr; bool hasnulls = HeapTupleHasNulls(tuple); int tdesc_natts = tupleDesc->natts; int natts; /* number of atts to extract */ @@ -1354,70 +1253,91 @@ heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, char *tp; /* ptr to tuple data */ uint32 off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ - bool slow = false; /* can we use/set attcacheoff? */ + int cacheoffattrs; + int firstnullattr; natts = HeapTupleHeaderGetNatts(tup); + /* Did someone forget to call TupleDescFinalize()? */ + Assert(tupleDesc->firstNonCachedOffAttr >= 0); + /* * In inheritance situations, it is possible that the given tuple actually * has more fields than the caller is expecting. Don't run off the end of * the caller's arrays. */ natts = Min(natts, tdesc_natts); + cacheoffattrs = Min(tupleDesc->firstNonCachedOffAttr, natts); + + if (hasnulls) + { + firstnullattr = first_null_attr(bp, natts); + cacheoffattrs = Min(cacheoffattrs, firstnullattr); + } + else + firstnullattr = natts; tp = (char *) tup + tup->t_hoff; + attnum = 0; - off = 0; + if (cacheoffattrs > 0) + { +#ifdef USE_ASSERT_CHECKING + /* In Assert enabled builds, verify attcacheoff is correct */ + off = 0; +#endif + do + { + cattr = TupleDescCompactAttr(tupleDesc, attnum); + +#ifdef USE_ASSERT_CHECKING + off = att_nominal_alignby(off, cattr->attalignby); + Assert(off == cattr->attcacheoff); + off += cattr->attlen; +#endif - for (attnum = 0; attnum < natts; attnum++) + values[attnum] = fetch_att(tp + cattr->attcacheoff, + cattr->attbyval, + cattr->attlen); + isnull[attnum] = false; + } while (++attnum < cacheoffattrs); + off = cattr->attcacheoff + cattr->attlen; + } + else + off = 0; + + for (; attnum < firstnullattr; attnum++) { - CompactAttribute *thisatt = TupleDescCompactAttr(tupleDesc, attnum); + cattr = TupleDescCompactAttr(tupleDesc, attnum); - if (hasnulls && att_isnull(attnum, bp)) - { - values[attnum] = (Datum) 0; - isnull[attnum] = true; - slow = true; /* can't use attcacheoff anymore */ - continue; - } + off = att_pointer_alignby(off, cattr->attalignby, cattr->attlen, + tp + off); isnull[attnum] = false; + values[attnum] = fetchatt(cattr, tp + off); - if (!slow && thisatt->attcacheoff >= 0) - off = thisatt->attcacheoff; - else if (thisatt->attlen == -1) - { - /* - * We can only cache the offset for a varlena attribute if the - * offset is already suitably aligned, so that there would be no - * pad bytes in any case: then the offset will be valid for either - * an aligned or unaligned value. - */ - if (!slow && - off == att_nominal_alignby(off, thisatt->attalignby)) - thisatt->attcacheoff = off; - else - { - off = att_pointer_alignby(off, thisatt->attalignby, -1, - tp + off); - slow = true; - } - } - else - { - /* not varlena, so safe to use att_nominal_alignby */ - off = att_nominal_alignby(off, thisatt->attalignby); + off = att_addlength_pointer(off, cattr->attlen, tp + off); + } + + for (; attnum < natts; attnum++) + { + Assert(hasnulls); - if (!slow) - thisatt->attcacheoff = off; + if (att_isnull(attnum, bp)) + { + values[attnum] = (Datum) 0; + isnull[attnum] = true; + continue; } - values[attnum] = fetchatt(thisatt, tp + off); + cattr = TupleDescCompactAttr(tupleDesc, attnum); + off = att_pointer_alignby(off, cattr->attalignby, cattr->attlen, + tp + off); - off = att_addlength_pointer(off, thisatt->attlen, tp + off); + isnull[attnum] = false; + values[attnum] = fetchatt(cattr, tp + off); - if (thisatt->attlen <= 0) - slow = true; /* can't use attcacheoff anymore */ + off = att_addlength_pointer(off, cattr->attlen, tp + off); } /* diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c index d7c8c53fd8d..084e0937a60 100644 --- a/src/backend/access/common/indextuple.c +++ b/src/backend/access/common/indextuple.c @@ -223,18 +223,6 @@ index_form_tuple_context(TupleDesc tupleDescriptor, * * This gets called from index_getattr() macro, and only in cases * where we can't use cacheoffset and the value is not null. - * - * This caches attribute offsets in the attribute descriptor. - * - * An alternative way to speed things up would be to cache offsets - * with the tuple, but that seems more difficult unless you take - * the storage hit of actually putting those offsets into the - * tuple you send to disk. Yuck. - * - * This scheme will be slightly slower than that, but should - * perform well for queries which hit large #'s of tuples. After - * you cache the offsets once, examining all the other tuples using - * the same attribute descriptor will go much quicker. -cim 5/4/91 * ---------------- */ Datum @@ -242,205 +230,129 @@ nocache_index_getattr(IndexTuple tup, int attnum, TupleDesc tupleDesc) { + CompactAttribute *cattr; char *tp; /* ptr to data part of tuple */ bits8 *bp = NULL; /* ptr to null bitmap in tuple */ - bool slow = false; /* do we have to walk attrs? */ int data_off; /* tuple data offset */ int off; /* current offset within data */ + int startAttr; + int firstnullattr; + bool hasnulls = IndexTupleHasNulls(tup); + int i; - /* ---------------- - * Three cases: - * - * 1: No nulls and no variable-width attributes. - * 2: Has a null or a var-width AFTER att. - * 3: Has nulls or var-widths BEFORE att. - * ---------------- - */ - - data_off = IndexInfoFindDataOffset(tup->t_info); + /* Did someone forget to call TupleDescFinalize()? */ + Assert(tupleDesc->firstNonCachedOffAttr >= 0); attnum--; - if (IndexTupleHasNulls(tup)) - { - /* - * there's a null somewhere in the tuple - * - * check to see if desired att is null - */ + /* + * If there are no NULLs before the required attnum, then we can start at + * the highest attribute with a known offset, or the first attribute if + * none have a cached offset. If the tuple has no variable width types, + * which is common with indexes, then we can use a slightly cheaper method + * of offset calculation, as we just need to add the attlen to the aligned + * offset when skipping over columns. When the tuple contains + * variable-width types, we must use att_addlength_pointer(), which does a + * bit more branching and is slightly less efficient. + */ + data_off = IndexInfoFindDataOffset(tup->t_info); + tp = (char *) tup + data_off; - /* XXX "knows" t_bits are just after fixed tuple header! */ + /* + * Find the first NULL column, or if there's none set the first NULL to + * attnum so that we can forego NULL checking all the way to attnum. + */ + if (hasnulls) + { bp = (bits8 *) ((char *) tup + sizeof(IndexTupleData)); - - /* - * Now check to see if any preceding bits are null... - */ - { - int byte = attnum >> 3; - int finalbit = attnum & 0x07; - - /* check for nulls "before" final bit of last byte */ - if ((~bp[byte]) & ((1 << finalbit) - 1)) - slow = true; - else - { - /* check for nulls in any "earlier" bytes */ - int i; - - for (i = 0; i < byte; i++) - { - if (bp[i] != 0xFF) - { - slow = true; - break; - } - } - } - } + firstnullattr = first_null_attr(bp, attnum); } + else + firstnullattr = attnum; - tp = (char *) tup + data_off; - - if (!slow) + if (tupleDesc->firstNonCachedOffAttr > 0) { - CompactAttribute *att; - - /* - * If we get here, there are no nulls up to and including the target - * attribute. If we have a cached offset, we can use it. - */ - att = TupleDescCompactAttr(tupleDesc, attnum); - if (att->attcacheoff >= 0) - return fetchatt(att, tp + att->attcacheoff); - - /* - * Otherwise, check for non-fixed-length attrs up to and including - * target. If there aren't any, it's safe to cheaply initialize the - * cached offsets for these attrs. - */ - if (IndexTupleHasVarwidths(tup)) - { - int j; - - for (j = 0; j <= attnum; j++) - { - if (TupleDescCompactAttr(tupleDesc, j)->attlen <= 0) - { - slow = true; - break; - } - } - } + startAttr = Min(tupleDesc->firstNonCachedOffAttr - 1, firstnullattr); + off = TupleDescCompactAttr(tupleDesc, startAttr)->attcacheoff; } - - if (!slow) + else { - int natts = tupleDesc->natts; - int j = 1; - - /* - * If we get here, we have a tuple with no nulls or var-widths up to - * and including the target attribute, so we can use the cached offset - * ... only we don't have it yet, or we'd not have got here. Since - * it's cheap to compute offsets for fixed-width columns, we take the - * opportunity to initialize the cached offsets for *all* the leading - * fixed-width columns, in hope of avoiding future visits to this - * routine. - */ - TupleDescCompactAttr(tupleDesc, 0)->attcacheoff = 0; + startAttr = 0; + off = 0; + } - /* we might have set some offsets in the slow path previously */ - while (j < natts && TupleDescCompactAttr(tupleDesc, j)->attcacheoff > 0) - j++; + /* Handle tuples with var-width attributes */ + if (IndexTupleHasVarwidths(tup)) + { + /* Calculate the offset up until the first NULL */ + for (i = startAttr; i < firstnullattr; i++) + { + cattr = TupleDescCompactAttr(tupleDesc, i); - off = TupleDescCompactAttr(tupleDesc, j - 1)->attcacheoff + - TupleDescCompactAttr(tupleDesc, j - 1)->attlen; + off = att_pointer_alignby(off, + cattr->attalignby, + cattr->attlen, + tp + off); + off = att_addlength_pointer(off, cattr->attlen, tp + off); + } - for (; j < natts; j++) + /* Calculate the offset for any remaining columns. */ + for (; i < attnum; i++) { - CompactAttribute *att = TupleDescCompactAttr(tupleDesc, j); + Assert(hasnulls); - if (att->attlen <= 0) - break; + if (att_isnull(i, bp)) + continue; - off = att_nominal_alignby(off, att->attalignby); + cattr = TupleDescCompactAttr(tupleDesc, i); - att->attcacheoff = off; - - off += att->attlen; + off = att_pointer_alignby(off, + cattr->attalignby, + cattr->attlen, + tp + off); + off = att_addlength_pointer(off, cattr->attlen, tp + off); } - - Assert(j > attnum); - - off = TupleDescCompactAttr(tupleDesc, attnum)->attcacheoff; } else { - bool usecache = true; - int i; + /* Handle tuples with only fixed-width attributes */ - /* - * Now we know that we have to walk the tuple CAREFULLY. But we still - * might be able to cache some offsets for next time. - * - * Note - This loop is a little tricky. For each non-null attribute, - * we have to first account for alignment padding before the attr, - * then advance over the attr based on its length. Nulls have no - * storage and no alignment padding either. We can use/set - * attcacheoff until we reach either a null or a var-width attribute. - */ - off = 0; - for (i = 0;; i++) /* loop exit is at "break" */ + /* Calculate the offset up until the first NULL */ + for (i = startAttr; i < firstnullattr; i++) { - CompactAttribute *att = TupleDescCompactAttr(tupleDesc, i); - - if (IndexTupleHasNulls(tup) && att_isnull(i, bp)) - { - usecache = false; - continue; /* this cannot be the target att */ - } - - /* If we know the next offset, we can skip the rest */ - if (usecache && att->attcacheoff >= 0) - off = att->attcacheoff; - else if (att->attlen == -1) - { - /* - * We can only cache the offset for a varlena attribute if the - * offset is already suitably aligned, so that there would be - * no pad bytes in any case: then the offset will be valid for - * either an aligned or unaligned value. - */ - if (usecache && - off == att_nominal_alignby(off, att->attalignby)) - att->attcacheoff = off; - else - { - off = att_pointer_alignby(off, att->attalignby, -1, - tp + off); - usecache = false; - } - } - else - { - /* not varlena, so safe to use att_nominal_alignby */ - off = att_nominal_alignby(off, att->attalignby); + cattr = TupleDescCompactAttr(tupleDesc, i); + + Assert(cattr->attlen > 0); + off = att_pointer_alignby(off, + cattr->attalignby, + cattr->attlen, + tp + off); + off += cattr->attlen; + } - if (usecache) - att->attcacheoff = off; - } + /* Calculate the offset for any remaining columns. */ + for (; i < attnum; i++) + { + Assert(hasnulls); - if (i == attnum) - break; + if (att_isnull(i, bp)) + continue; - off = att_addlength_pointer(off, att->attlen, tp + off); + cattr = TupleDescCompactAttr(tupleDesc, i); - if (usecache && att->attlen <= 0) - usecache = false; + Assert(cattr->attlen > 0); + off = att_pointer_alignby(off, + cattr->attalignby, + cattr->attlen, + tp + off); + off += cattr->attlen; } } - return fetchatt(TupleDescCompactAttr(tupleDesc, attnum), tp + off); + cattr = TupleDescCompactAttr(tupleDesc, attnum); + off = att_pointer_alignby(off, cattr->attalignby, + cattr->attlen, tp + off); + return fetchatt(cattr, tp + off); } /* @@ -480,63 +392,86 @@ index_deform_tuple_internal(TupleDesc tupleDescriptor, Datum *values, bool *isnull, char *tp, bits8 *bp, int hasnulls) { + CompactAttribute *cattr; int natts = tupleDescriptor->natts; /* number of atts to extract */ - int attnum; + int attnum = 0; int off = 0; /* offset in tuple data */ - bool slow = false; /* can we use/set attcacheoff? */ + int cacheoffattrs; + int firstnullattr; /* Assert to protect callers who allocate fixed-size arrays */ Assert(natts <= INDEX_MAX_KEYS); - for (attnum = 0; attnum < natts; attnum++) + /* Did someone forget to call TupleDescFinalize()? */ + Assert(tupleDescriptor->firstNonCachedOffAttr >= 0); + + cacheoffattrs = Min(tupleDescriptor->firstNonCachedOffAttr, natts); + + if (hasnulls) + { + firstnullattr = first_null_attr(bp, natts); + cacheoffattrs = Min(cacheoffattrs, firstnullattr); + } + else + firstnullattr = natts; + + if (cacheoffattrs > 0) { - CompactAttribute *thisatt = TupleDescCompactAttr(tupleDescriptor, attnum); +#ifdef USE_ASSERT_CHECKING + /* In Assert enabled builds, verify attcacheoff is correct */ + off = 0; +#endif - if (hasnulls && att_isnull(attnum, bp)) + do { - values[attnum] = (Datum) 0; - isnull[attnum] = true; - slow = true; /* can't use attcacheoff anymore */ - continue; - } + cattr = TupleDescCompactAttr(tupleDescriptor, attnum); + +#ifdef USE_ASSERT_CHECKING + off = att_nominal_alignby(off, cattr->attalignby); + Assert(off == cattr->attcacheoff); + off += cattr->attlen; +#endif + + values[attnum] = fetch_att(tp + cattr->attcacheoff, cattr->attbyval, + cattr->attlen); + isnull[attnum] = false; + } while (++attnum < cacheoffattrs); + + off = cattr->attcacheoff + cattr->attlen; + } + + for (; attnum < firstnullattr; attnum++) + { + cattr = TupleDescCompactAttr(tupleDescriptor, attnum); + + off = att_pointer_alignby(off, cattr->attalignby, cattr->attlen, + tp + off); isnull[attnum] = false; + values[attnum] = fetchatt(cattr, tp + off); - if (!slow && thisatt->attcacheoff >= 0) - off = thisatt->attcacheoff; - else if (thisatt->attlen == -1) - { - /* - * We can only cache the offset for a varlena attribute if the - * offset is already suitably aligned, so that there would be no - * pad bytes in any case: then the offset will be valid for either - * an aligned or unaligned value. - */ - if (!slow && - off == att_nominal_alignby(off, thisatt->attalignby)) - thisatt->attcacheoff = off; - else - { - off = att_pointer_alignby(off, thisatt->attalignby, -1, - tp + off); - slow = true; - } - } - else - { - /* not varlena, so safe to use att_nominal_alignby */ - off = att_nominal_alignby(off, thisatt->attalignby); + off = att_addlength_pointer(off, cattr->attlen, tp + off); + } + + for (; attnum < natts; attnum++) + { + Assert(hasnulls); - if (!slow) - thisatt->attcacheoff = off; + if (att_isnull(attnum, bp)) + { + values[attnum] = (Datum) 0; + isnull[attnum] = true; + continue; } - values[attnum] = fetchatt(thisatt, tp + off); + cattr = TupleDescCompactAttr(tupleDescriptor, attnum); + off = att_pointer_alignby(off, cattr->attalignby, cattr->attlen, + tp + off); - off = att_addlength_pointer(off, thisatt->attlen, tp + off); + isnull[attnum] = false; + values[attnum] = fetchatt(cattr, tp + off); - if (thisatt->attlen <= 0) - slow = true; /* can't use attcacheoff anymore */ + off = att_addlength_pointer(off, cattr->attlen, tp + off); } } diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index e98de806a77..25364db630a 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -214,6 +214,9 @@ CreateTemplateTupleDesc(int natts) desc->tdtypmod = -1; desc->tdrefcount = -1; /* assume not reference-counted */ + /* This will be set to the correct value by TupleDescFinalize() */ + desc->firstNonCachedOffAttr = -1; + return desc; } @@ -474,6 +477,9 @@ TupleDescCopy(TupleDesc dst, TupleDesc src) * descriptor to another. * * !!! Constraints and defaults are not copied !!! + * + * The caller must take care of calling TupleDescFinalize() on once all + * TupleDesc changes have been made. */ void TupleDescCopyEntry(TupleDesc dst, AttrNumber dstAttno, @@ -506,6 +512,37 @@ TupleDescCopyEntry(TupleDesc dst, AttrNumber dstAttno, populate_compact_attribute(dst, dstAttno - 1); } +/* + * TupleDescFinalize + * Finalize the given TupleDesc. This must be called after the + * attributes arrays have been populated or adjusted by any code. + * + * Must be called after populate_compact_attribute() and before + * BlessTupleDesc(). + */ +void +TupleDescFinalize(TupleDesc tupdesc) +{ + int firstNonCachedOffAttr = 0; + int offp = 0; + + for (int i = 0; i < tupdesc->natts; i++) + { + CompactAttribute *cattr = TupleDescCompactAttr(tupdesc, i); + + if (cattr->attlen <= 0) + break; + + offp = att_nominal_alignby(offp, cattr->attalignby); + cattr->attcacheoff = offp; + + offp += cattr->attlen; + firstNonCachedOffAttr = i + 1; + } + + tupdesc->firstNonCachedOffAttr = firstNonCachedOffAttr; +} + /* * Free a TupleDesc including all substructure */ diff --git a/src/backend/access/spgist/spgutils.c b/src/backend/access/spgist/spgutils.c index b246e8127db..a4694bd8065 100644 --- a/src/backend/access/spgist/spgutils.c +++ b/src/backend/access/spgist/spgutils.c @@ -335,9 +335,6 @@ getSpGistTupleDesc(Relation index, SpGistTypeDesc *keyType) /* We shouldn't need to bother with making these valid: */ att->attcompression = InvalidCompressionMethod; att->attcollation = InvalidOid; - /* In case we changed typlen, we'd better reset following offsets */ - for (int i = spgFirstIncludeColumn; i < outTupDesc->natts; i++) - TupleDescCompactAttr(outTupDesc, i)->attcacheoff = -1; populate_compact_attribute(outTupDesc, spgKeyColumn); TupleDescFinalize(outTupDesc); diff --git a/src/backend/executor/execTuples.c b/src/backend/executor/execTuples.c index e6ab51e6404..89f18be5d82 100644 --- a/src/backend/executor/execTuples.c +++ b/src/backend/executor/execTuples.c @@ -992,118 +992,6 @@ tts_buffer_heap_store_tuple(TupleTableSlot *slot, HeapTuple tuple, } } -/* - * slot_deform_heap_tuple_internal - * An always inline helper function for use in slot_deform_heap_tuple to - * allow the compiler to emit specialized versions of this function for - * various combinations of "slow" and "hasnulls". For example, if a - * given tuple has no nulls, then we needn't check "hasnulls" for every - * attribute that we're deforming. The caller can just call this - * function with hasnulls set to constant-false and have the compiler - * remove the constant-false branches and emit more optimal code. - * - * Returns the next attnum to deform, which can be equal to natts when the - * function manages to deform all requested attributes. *offp is an input and - * output parameter which is the byte offset within the tuple to start deforming - * from which, on return, gets set to the offset where the next attribute - * should be deformed from. *slowp is set to true when subsequent deforming - * of this tuple must use a version of this function with "slow" passed as - * true. - * - * Callers cannot assume when we return "attnum" (i.e. all requested - * attributes have been deformed) that slow mode isn't required for any - * additional deforming as the final attribute may have caused a switch to - * slow mode. - */ -static pg_attribute_always_inline int -slot_deform_heap_tuple_internal(TupleTableSlot *slot, HeapTuple tuple, - int attnum, int natts, bool slow, - bool hasnulls, uint32 *offp, bool *slowp) -{ - TupleDesc tupleDesc = slot->tts_tupleDescriptor; - Datum *values = slot->tts_values; - bool *isnull = slot->tts_isnull; - HeapTupleHeader tup = tuple->t_data; - char *tp; /* ptr to tuple data */ - bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ - bool slownext = false; - - tp = (char *) tup + tup->t_hoff; - - for (; attnum < natts; attnum++) - { - CompactAttribute *thisatt = TupleDescCompactAttr(tupleDesc, attnum); - - if (hasnulls && att_isnull(attnum, bp)) - { - values[attnum] = (Datum) 0; - isnull[attnum] = true; - if (!slow) - { - *slowp = true; - return attnum + 1; - } - else - continue; - } - - isnull[attnum] = false; - - /* calculate the offset of this attribute */ - if (!slow && thisatt->attcacheoff >= 0) - *offp = thisatt->attcacheoff; - else if (thisatt->attlen == -1) - { - /* - * We can only cache the offset for a varlena attribute if the - * offset is already suitably aligned, so that there would be no - * pad bytes in any case: then the offset will be valid for either - * an aligned or unaligned value. - */ - if (!slow && *offp == att_nominal_alignby(*offp, thisatt->attalignby)) - thisatt->attcacheoff = *offp; - else - { - *offp = att_pointer_alignby(*offp, - thisatt->attalignby, - -1, - tp + *offp); - - if (!slow) - slownext = true; - } - } - else - { - /* not varlena, so safe to use att_nominal_alignby */ - *offp = att_nominal_alignby(*offp, thisatt->attalignby); - - if (!slow) - thisatt->attcacheoff = *offp; - } - - values[attnum] = fetchatt(thisatt, tp + *offp); - - *offp = att_addlength_pointer(*offp, thisatt->attlen, tp + *offp); - - /* check if we need to switch to slow mode */ - if (!slow) - { - /* - * We're unable to deform any further if the above code set - * 'slownext', or if this isn't a fixed-width attribute. - */ - if (slownext || thisatt->attlen <= 0) - { - *slowp = true; - return attnum + 1; - } - } - } - - return natts; -} - /* * slot_deform_heap_tuple * Given a TupleTableSlot, extract data from the slot's physical tuple @@ -1122,78 +1010,140 @@ static pg_attribute_always_inline void slot_deform_heap_tuple(TupleTableSlot *slot, HeapTuple tuple, uint32 *offp, int natts) { + CompactAttribute *cattr; + TupleDesc tupleDesc = slot->tts_tupleDescriptor; bool hasnulls = HeapTupleHasNulls(tuple); + HeapTupleHeader tup = tuple->t_data; + bits8 *bp; /* ptr to null bitmap in tuple */ int attnum; + int firstNonCacheOffsetAttr; + int firstNullAttr; + Datum *values; + bool *isnull; + char *tp; /* ptr to tuple data */ uint32 off; /* offset in tuple data */ - bool slow; /* can we use/set attcacheoff? */ + + /* Did someone forget to call TupleDescFinalize()? */ + Assert(tupleDesc->firstNonCachedOffAttr >= 0); /* We can only fetch as many attributes as the tuple has. */ - natts = Min(HeapTupleHeaderGetNatts(tuple->t_data), natts); + natts = Min(HeapTupleHeaderGetNatts(tup), natts); + attnum = slot->tts_nvalid; + firstNonCacheOffsetAttr = Min(tupleDesc->firstNonCachedOffAttr, natts); + + if (hasnulls) + { + bp = tup->t_bits; + firstNullAttr = first_null_attr(bp, natts); + firstNonCacheOffsetAttr = Min(firstNonCacheOffsetAttr, firstNullAttr); + } + else + { + bp = NULL; + firstNullAttr = natts; + } + + values = slot->tts_values; + isnull = slot->tts_isnull; + tp = (char *) tup + tup->t_hoff; /* - * Check whether the first call for this tuple, and initialize or restore - * loop state. + * Handle the portion of the tuple that we have cached the offset for up + * to the first NULL attribute. The offset is effectively fixed for these + * so we can use the CompactAttribute's attcacheoff. */ - attnum = slot->tts_nvalid; - if (attnum == 0) + if (attnum < firstNonCacheOffsetAttr) + { +#ifdef USE_ASSERT_CHECKING + int offcheck; + + /* In Assert enabled builds, verify attcacheoff is correct */ + if (attnum == 0) + offcheck = 0; + else + offcheck = *offp; +#endif + do + { + cattr = TupleDescCompactAttr(tupleDesc, attnum); + +#ifdef USE_ASSERT_CHECKING + offcheck = att_nominal_alignby(offcheck, cattr->attalignby); + Assert(offcheck == cattr->attcacheoff); + offcheck += cattr->attlen; +#endif + + values[attnum] = fetchatt(cattr, tp + cattr->attcacheoff); + isnull[attnum] = false; + } while (++attnum < firstNonCacheOffsetAttr); + + /* + * Point the offset after the end of the last attribute with a cached + * offset. We expect the final cached offset attribute to have a + * fixed width, so just add the attlen to the attcacheoff + */ + Assert(cattr->attlen > 0); + off = cattr->attcacheoff + cattr->attlen; + } + else if (attnum == 0) { /* Start from the first attribute */ off = 0; - slow = false; } else { /* Restore state from previous execution */ off = *offp; - slow = TTS_SLOW(slot); } /* - * If 'slow' isn't set, try deforming using deforming code that does not - * contain any of the extra checks required for non-fixed offset - * deforming. During deforming, if or when we find a NULL or a variable - * length attribute, we'll switch to a deforming method which includes the - * extra code required for non-fixed offset deforming, a.k.a slow mode. - * Because this is performance critical, we inline - * slot_deform_heap_tuple_internal passing the 'slow' and 'hasnull' - * parameters as constants to allow the compiler to emit specialized code - * with the known-const false comparisons and subsequent branches removed. + * Handle any portion of the tuple that doesn't have a fixed offset up + * until the first NULL attribute. This loops only differs from the one + * after it by the NULL checks. */ - if (!slow) + for (; attnum < firstNullAttr; attnum++) { - /* Tuple without any NULLs? We can skip doing any NULL checking */ - if (!hasnulls) - attnum = slot_deform_heap_tuple_internal(slot, - tuple, - attnum, - natts, - false, /* slow */ - false, /* hasnulls */ - &off, - &slow); - else - attnum = slot_deform_heap_tuple_internal(slot, - tuple, - attnum, - natts, - false, /* slow */ - true, /* hasnulls */ - &off, - &slow); + cattr = TupleDescCompactAttr(tupleDesc, attnum); + + /* align the offset for this attribute */ + off = att_pointer_alignby(off, + cattr->attalignby, + cattr->attlen, + tp + off); + + values[attnum] = fetchatt(cattr, tp + off); + isnull[attnum] = false; + + /* move the offset beyond this attribute */ + off = att_addlength_pointer(off, cattr->attlen, tp + off); } - /* If there's still work to do then we must be in slow mode */ - if (attnum < natts) + /* + * Now handle any remaining tuples, this time include NULL checks as we're + * now at the first NULL attribute. + */ + for (; attnum < natts; attnum++) { - /* XXX is it worth adding a separate call when hasnulls is false? */ - attnum = slot_deform_heap_tuple_internal(slot, - tuple, - attnum, - natts, - true, /* slow */ - hasnulls, - &off, - &slow); + if (att_isnull(attnum, bp)) + { + values[attnum] = (Datum) 0; + isnull[attnum] = true; + continue; + } + + cattr = TupleDescCompactAttr(tupleDesc, attnum); + + /* align the offset for this attribute */ + off = att_pointer_alignby(off, + cattr->attalignby, + cattr->attlen, + tp + off); + + values[attnum] = fetchatt(cattr, tp + off); + isnull[attnum] = false; + + /* move the offset beyond this attribute */ + off = att_addlength_pointer(off, cattr->attlen, tp + off); } /* @@ -1201,10 +1151,6 @@ slot_deform_heap_tuple(TupleTableSlot *slot, HeapTuple tuple, uint32 *offp, */ slot->tts_nvalid = attnum; *offp = off; - if (slow) - slot->tts_flags |= TTS_FLAG_SLOW; - else - slot->tts_flags &= ~TTS_FLAG_SLOW; } const TupleTableSlotOps TTSOpsVirtual = { @@ -2259,10 +2205,16 @@ ExecTypeSetColNames(TupleDesc typeInfo, List *namesList) * This happens "for free" if the tupdesc came from a relcache entry, but * not if we have manufactured a tupdesc for a transient RECORD datatype. * In that case we have to notify typcache.c of the existence of the type. + * + * TupleDescFinalize() must be called on the TupleDesc before calling this + * function. */ TupleDesc BlessTupleDesc(TupleDesc tupdesc) { + /* Did someone forget to call TupleDescFinalize()? */ + Assert(tupdesc->firstNonCachedOffAttr >= 0); + if (tupdesc->tdtypeid == RECORDOID && tupdesc->tdtypmod < 0) assign_record_type_typmod(tupdesc); diff --git a/src/backend/jit/llvm/llvmjit_deform.c b/src/backend/jit/llvm/llvmjit_deform.c index 3eb087eb56b..12521e3e46a 100644 --- a/src/backend/jit/llvm/llvmjit_deform.c +++ b/src/backend/jit/llvm/llvmjit_deform.c @@ -62,7 +62,6 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, LLVMValueRef v_tts_values; LLVMValueRef v_tts_nulls; LLVMValueRef v_slotoffp; - LLVMValueRef v_flagsp; LLVMValueRef v_nvalidp; LLVMValueRef v_nvalid; LLVMValueRef v_maxatt; @@ -178,7 +177,6 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_tts_nulls = l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL, "tts_ISNULL"); - v_flagsp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, ""); v_nvalidp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, ""); if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple) @@ -747,14 +745,10 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, { LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); - LLVMValueRef v_flags; LLVMBuildStore(b, l_int16_const(lc, natts), v_nvalidp); v_off = LLVMBuildTrunc(b, v_off, LLVMInt32TypeInContext(lc), ""); LLVMBuildStore(b, v_off, v_slotoffp); - v_flags = l_load(b, LLVMInt16TypeInContext(lc), v_flagsp, "tts_flags"); - v_flags = LLVMBuildOr(b, v_flags, l_int16_const(lc, TTS_FLAG_SLOW), ""); - LLVMBuildStore(b, v_flags, v_flagsp); LLVMBuildRetVoid(b); } diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 770edb34e08..998be24ac41 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -666,14 +666,6 @@ RelationBuildTupleDesc(Relation relation) elog(ERROR, "pg_attribute catalog is missing %d attribute(s) for relation OID %u", need, RelationGetRelid(relation)); - /* - * We can easily set the attcacheoff value for the first attribute: it - * must be zero. This eliminates the need for special cases for attnum=1 - * that used to exist in fastgetattr() and index_getattr(). - */ - if (RelationGetNumberOfAttributes(relation) > 0) - TupleDescCompactAttr(relation->rd_att, 0)->attcacheoff = 0; - /* * Set up constraint/default info */ @@ -1985,8 +1977,6 @@ formrdesc(const char *relationName, Oid relationReltype, populate_compact_attribute(relation->rd_att, i); } - /* initialize first attribute's attcacheoff, cf RelationBuildTupleDesc */ - TupleDescCompactAttr(relation->rd_att, 0)->attcacheoff = 0; TupleDescFinalize(relation->rd_att); /* mark not-null status */ @@ -4446,8 +4436,6 @@ BuildHardcodedDescriptor(int natts, const FormData_pg_attribute *attrs) populate_compact_attribute(result, i); } - /* initialize first attribute's attcacheoff, cf RelationBuildTupleDesc */ - TupleDescCompactAttr(result, 0)->attcacheoff = 0; TupleDescFinalize(result); /* Note: we don't bother to set up a TupleConstr entry */ diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index d406825ff22..94b4279b7f1 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -865,20 +865,17 @@ extern MinimalTuple minimal_expand_tuple(HeapTuple sourceTuple, TupleDesc tupleD static inline Datum fastgetattr(HeapTuple tup, int attnum, TupleDesc tupleDesc, bool *isnull) { - Assert(attnum > 0); + CompactAttribute *att = TupleDescCompactAttr(tupleDesc, attnum - 1); + Assert(attnum > 0); *isnull = false; - if (HeapTupleNoNulls(tup)) - { - CompactAttribute *att; - att = TupleDescCompactAttr(tupleDesc, attnum - 1); - if (att->attcacheoff >= 0) - return fetchatt(att, (char *) tup->t_data + tup->t_data->t_hoff + - att->attcacheoff); - else - return nocachegetattr(tup, attnum, tupleDesc); - } + if (att->attcacheoff >= 0 && !HeapTupleHasNulls(tup)) + return fetchatt(att, (char *) tup->t_data + tup->t_data->t_hoff + + att->attcacheoff); + + if (HeapTupleNoNulls(tup)) + return nocachegetattr(tup, attnum, tupleDesc); else { if (att_isnull(attnum - 1, tup->t_data->t_bits)) diff --git a/src/include/access/itup.h b/src/include/access/itup.h index 57e4daafb0d..e4bb27b7e58 100644 --- a/src/include/access/itup.h +++ b/src/include/access/itup.h @@ -131,24 +131,20 @@ IndexInfoFindDataOffset(unsigned short t_info) static inline Datum index_getattr(IndexTuple tup, int attnum, TupleDesc tupleDesc, bool *isnull) { + CompactAttribute *attr = TupleDescCompactAttr(tupleDesc, attnum - 1); + Assert(isnull); Assert(attnum > 0); *isnull = false; - if (!IndexTupleHasNulls(tup)) - { - CompactAttribute *attr = TupleDescCompactAttr(tupleDesc, attnum - 1); + if (attr->attcacheoff >= 0 && !IndexTupleHasNulls(tup)) + return fetchatt(attr, + (char *) tup + IndexInfoFindDataOffset(tup->t_info) + + attr->attcacheoff); - if (attr->attcacheoff >= 0) - { - return fetchatt(attr, - (char *) tup + IndexInfoFindDataOffset(tup->t_info) + - attr->attcacheoff); - } - else - return nocache_index_getattr(tup, attnum, tupleDesc); - } + if (!IndexTupleHasNulls(tup)) + return nocache_index_getattr(tup, attnum, tupleDesc); else { if (att_isnull(attnum - 1, (bits8 *) tup + sizeof(IndexTupleData))) diff --git a/src/include/access/tupdesc.h b/src/include/access/tupdesc.h index 595413dbbc5..99d9017d1a6 100644 --- a/src/include/access/tupdesc.h +++ b/src/include/access/tupdesc.h @@ -131,6 +131,12 @@ typedef struct CompactAttribute * Any code making changes manually to and fields in the FormData_pg_attribute * array must subsequently call populate_compact_attribute() to flush the * changes out to the corresponding 'compact_attrs' element. + * + * firstNonCachedOffAttr stores the index into the compact_attrs array for the + * first attribute that we don't have a known attcacheoff for. + * + * Once a TupleDesc has been populated, before it is used for any purpose + * TupleDescFinalize() must be called on it. */ typedef struct TupleDescData { @@ -138,6 +144,8 @@ typedef struct TupleDescData Oid tdtypeid; /* composite type ID for tuple type */ int32 tdtypmod; /* typmod for tuple type */ int tdrefcount; /* reference count, or -1 if not counting */ + int firstNonCachedOffAttr; /* index of the first att without an + * attcacheoff */ TupleConstr *constr; /* constraints, or NULL if none */ /* compact_attrs[N] is the compact metadata of Attribute Number N+1 */ CompactAttribute compact_attrs[FLEXIBLE_ARRAY_MEMBER]; @@ -195,7 +203,6 @@ extern TupleDesc CreateTupleDescTruncatedCopy(TupleDesc tupdesc, int natts); extern TupleDesc CreateTupleDescCopyConstr(TupleDesc tupdesc); -#define TupleDescFinalize(d) ((void) 0) #define TupleDescSize(src) \ (offsetof(struct TupleDescData, compact_attrs) + \ (src)->natts * sizeof(CompactAttribute) + \ @@ -206,6 +213,7 @@ extern void TupleDescCopy(TupleDesc dst, TupleDesc src); extern void TupleDescCopyEntry(TupleDesc dst, AttrNumber dstAttno, TupleDesc src, AttrNumber srcAttno); +extern void TupleDescFinalize(TupleDesc tupdesc); extern void FreeTupleDesc(TupleDesc tupdesc); extern void IncrTupleDescRefCount(TupleDesc tupdesc); diff --git a/src/include/access/tupmacs.h b/src/include/access/tupmacs.h index e6df8264750..fcaf6ad149f 100644 --- a/src/include/access/tupmacs.h +++ b/src/include/access/tupmacs.h @@ -15,6 +15,7 @@ #define TUPMACS_H #include "catalog/pg_type_d.h" /* for TYPALIGN macros */ +#include "port/pg_bitutils.h" /* @@ -69,6 +70,70 @@ fetch_att(const void *T, bool attbyval, int attlen) else return PointerGetDatum(T); } + +/* + * first_null_attr + * Inspect a NULL bitmask from a tuple and return the 0-based attnum of the + * first NULL attribute. Returns natts if no NULLs were found. + * + * We expect that 'bits' contains at least one 0 bit somewhere in the mask, + * not necessarily < natts. + */ +static inline int +first_null_attr(const bits8 *bits, int natts) +{ + int lastByte = natts >> 3; + uint8 mask; + int res = natts; + uint8 byte; + +#ifdef USE_ASSERT_CHECKING + int firstnull_check = natts; + + /* Do it the slow way and check we get the same answer. */ + for (int i = 0; i < natts; i++) + { + if (att_isnull(i, bits)) + { + firstnull_check = i; + break; + } + } +#endif + + /* Process all bytes up to just before the byte for the natts index */ + for (int bytenum = 0; bytenum < lastByte; bytenum++) + { + if (bits[bytenum] != 0xFF) + { + byte = ~bits[bytenum]; + res = bytenum << 3; + res += pg_rightmost_one_pos[byte]; + + Assert(res == firstnull_check); + return res; + } + } + + /* + * Create a mask with all bits beyond natts's bit set to off. This + * assumes the code above will have found a 0-bit before we run off the + * end of the bits array. Tuples without any NULLs won't have a bitmask + * to mark NULLs. + */ + mask = 0xFF & ((((uint8) 1) << (natts & 7)) - 1); + byte = (~bits[lastByte]) & mask; + + if (byte != 0) + { + res = lastByte << 3; + res += pg_rightmost_one_pos[byte]; + } + + Assert(res == firstnull_check); + + return res; +} #endif /* FRONTEND */ /* diff --git a/src/include/executor/tuptable.h b/src/include/executor/tuptable.h index a2dfd707e78..363c5f33697 100644 --- a/src/include/executor/tuptable.h +++ b/src/include/executor/tuptable.h @@ -84,9 +84,6 @@ * tts_values/tts_isnull are allocated either when the slot is created (when * the descriptor is provided), or when a descriptor is assigned to the slot; * they are of length equal to the descriptor's natts. - * - * The TTS_FLAG_SLOW flag is saved state for - * slot_deform_heap_tuple, and should not be touched by any other code. *---------- */ @@ -98,12 +95,8 @@ #define TTS_FLAG_SHOULDFREE (1 << 2) #define TTS_SHOULDFREE(slot) (((slot)->tts_flags & TTS_FLAG_SHOULDFREE) != 0) -/* saved state for slot_deform_heap_tuple */ -#define TTS_FLAG_SLOW (1 << 3) -#define TTS_SLOW(slot) (((slot)->tts_flags & TTS_FLAG_SLOW) != 0) - /* fixed tuple descriptor */ -#define TTS_FLAG_FIXED (1 << 4) +#define TTS_FLAG_FIXED (1 << 4) /* XXX change to #3? */ #define TTS_FIXED(slot) (((slot)->tts_flags & TTS_FLAG_FIXED) != 0) struct TupleTableSlotOps; -- 2.51.0