From c5ec896a2df02041f08d1e41a982223781137d5f Mon Sep 17 00:00:00 2001 From: David Rowley Date: Thu, 17 Nov 2022 13:04:49 +1300 Subject: [PATCH v2 2/5] Perform memory prefetching in heapgetpage --- src/backend/access/heap/heapam.c | 87 +++++++++++++++++++----- src/backend/access/heap/heapam_handler.c | 7 +- src/include/access/heapam.h | 1 + 3 files changed, 74 insertions(+), 21 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index d18c5ca6f5..81c7f69644 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -451,31 +451,82 @@ heapgetpage(TableScanDesc sscan, BlockNumber block) */ all_visible = PageIsAllVisible(page) && !snapshot->takenDuringRecovery; - for (lineoff = FirstOffsetNumber, lpp = PageGetItemId(page, lineoff); - lineoff <= lines; - lineoff++, lpp++) + if (all_visible) { - if (ItemIdIsNormal(lpp)) + HeapTupleData loctup; + + loctup.t_tableOid = RelationGetRelid(scan->rs_base.rs_rd); + + for (lineoff = FirstOffsetNumber, lpp = PageGetItemId(page, lineoff); + lineoff <= lines; + lineoff++, lpp++) { - HeapTupleData loctup; - bool valid; + if (!ItemIdIsNormal(lpp)) + continue; - loctup.t_tableOid = RelationGetRelid(scan->rs_base.rs_rd); loctup.t_data = (HeapTupleHeader) PageGetItem(page, lpp); loctup.t_len = ItemIdGetLength(lpp); ItemPointerSet(&(loctup.t_self), block, lineoff); - if (all_visible) - valid = true; - else - valid = HeapTupleSatisfiesVisibility(&loctup, snapshot, buffer); + HeapCheckForSerializableConflictOut(true, scan->rs_base.rs_rd, + &loctup, buffer, snapshot); + scan->rs_vistuples[ntup++] = lineoff; + } + scan->rs_startindex = 0; + } + else + { + HeapTupleData loctup; + int normcount = 0; + int startindex = MaxHeapTuplesPerPage; + OffsetNumber normoffsets[MaxHeapTuplesPerPage]; + + loctup.t_tableOid = RelationGetRelid(scan->rs_base.rs_rd); + + /* + * Iterate forward over line items, they're laid out in increasing + * order in memory. Doing this separately allows to benefit from + * out-of-order capabilities of the CPU and simplifies the next loop. + */ + for (lineoff = FirstOffsetNumber, lpp = PageGetItemId(page, lineoff); + lineoff <= lines; + lineoff++, lpp++) + { + pg_prefetch_mem(PageGetItemId(page, lineoff+5)); + if (ItemIdIsNormal(lpp)) + normoffsets[normcount++] = lineoff; + } + + /* + * Process tuples in reverse order. That'll most often lead to memory + * accesses in increasing order, which typically is more efficient for + * the CPUs prefetcher. To avoid affecting sort order, we populate the + * rs_vistuples[] array backwards and use rs_startindex to mark the + * first used element in the array. + */ + for (int i = normcount - 1; i >= 0; i--) + { + bool valid; + + lineoff = normoffsets[i]; + lpp = PageGetItemId(page, lineoff); + + loctup.t_data = (HeapTupleHeader) PageGetItem(page, lpp); + loctup.t_len = ItemIdGetLength(lpp); + ItemPointerSet(&(loctup.t_self), block, lineoff); + valid = HeapTupleSatisfiesVisibility(&loctup, snapshot, buffer); HeapCheckForSerializableConflictOut(valid, scan->rs_base.rs_rd, &loctup, buffer, snapshot); if (valid) - scan->rs_vistuples[ntup++] = lineoff; + { + scan->rs_vistuples[--startindex] = lineoff; + ntup++; + } } + /* record the first used element in rs_vistuples[] */ + scan->rs_startindex = startindex; } LockBuffer(buffer, BUFFER_LOCK_UNLOCK); @@ -902,7 +953,7 @@ heapgettup_pagemode(HeapScanDesc scan, else block = scan->rs_startblock; /* first page */ heapgetpage((TableScanDesc) scan, block); - lineindex = 0; + lineindex = scan->rs_startindex; scan->rs_inited = true; } else @@ -917,7 +968,7 @@ heapgettup_pagemode(HeapScanDesc scan, lines = scan->rs_ntuples; /* block and lineindex now reference the next visible tid */ - linesleft = lines - lineindex; + linesleft = lines - lineindex + scan->rs_startindex; } else if (backward) { @@ -968,7 +1019,7 @@ heapgettup_pagemode(HeapScanDesc scan, if (!scan->rs_inited) { - lineindex = lines - 1; + lineindex = scan->rs_startindex + lines - 1; scan->rs_inited = true; } else @@ -977,7 +1028,7 @@ heapgettup_pagemode(HeapScanDesc scan, } /* block and lineindex now reference the previous visible tid */ - linesleft = lineindex + 1; + linesleft = lineindex + 1 - scan->rs_startindex; } else { @@ -1127,9 +1178,9 @@ heapgettup_pagemode(HeapScanDesc scan, lines = scan->rs_ntuples; linesleft = lines; if (backward) - lineindex = lines - 1; + lineindex = MaxHeapTuplesPerPage - 1; else - lineindex = 0; + lineindex = scan->rs_startindex; } } diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index ab1bcf3522..d39284465b 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -2490,15 +2490,16 @@ SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, { /* * In pageatatime mode, heapgetpage() already did visibility checks, - * so just look at the info it left in rs_vistuples[]. + * so just look at the info it left in rs_vistuples[] starting at + * rs_startindex. * * We use a binary search over the known-sorted array. Note: we could * save some effort if we insisted that NextSampleTuple select tuples * in increasing order, but it's not clear that there would be enough * gain to justify the restriction. */ - int start = 0, - end = hscan->rs_ntuples - 1; + int start = hscan->rs_startindex, + end = hscan->rs_startindex + hscan->rs_ntuples - 1; while (start <= end) { diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 810baaf9d0..aba0795fc6 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -74,6 +74,7 @@ typedef struct HeapScanDescData /* these fields only used in page-at-a-time mode and for bitmap scans */ int rs_cindex; /* current tuple's index in vistuples */ int rs_ntuples; /* number of visible tuples on page */ + int rs_startindex; /* first used element in rs_vistuples[] */ OffsetNumber rs_vistuples[MaxHeapTuplesPerPage]; /* their offsets */ } HeapScanDescData; typedef struct HeapScanDescData *HeapScanDesc; -- 2.35.1.windows.2