Re: Upgrading rant.

On Fri, 03 Jan 2003 15:37:56 -0500, Tom Lane <tgl(at)sss(dot)pgh(dot)pa(dot)us> wrote:
>The system tables are not the problem. [...]
>
>Changes in the on-disk representation of user tables would be harder to
>deal with, but they are also much rarer (AFAIR we've only done that
>twice: WAL required additions to page and tuple headers, and then there
>were Manfred's space-saving changes in 7.3).

So I'm the bad guy? ;-) AFAICS handling the page and tuple format
changes doesn't need much more than what I have hacked together
yesterday afternoon:

#include <access/htup.h>

typedef struct HeapTupleHeader72Data
{
Oid t_oid; /* OID of this tuple -- 4 bytes */
CommandId t_cmin; /* insert CID stamp -- 4 bytes each */
CommandId t_cmax; /* delete CommandId stamp */
TransactionId t_xmin; /* insert XID stamp -- 4 bytes each */
TransactionId t_xmax; /* delete XID stamp */
ItemPointerData t_ctid; /* current TID of this or newer tuple */
int16 t_natts; /* number of attributes */
uint16 t_infomask; /* various infos */
uint8 t_hoff; /* sizeof() tuple header */
/* ^ - 31 bytes - ^ */
bits8 t_bits[1];
/* bit map of NULLs */
} HeapTupleHeader72Data;

typedef HeapTupleHeader72Data *HeapTupleHeader72;

/*
** Convert a pre-7.3 heap tuple header to 7.3 format.
**
** On entry ht points to a heap tuple header in 7.2 format,
** which will be converted to the new format in place.
** If compact is true, the size of the heap tuple header
** (t_hoff) is reduced, otherwise enough padding bytes are
** inserted to keep the old length.
**
** The return value is the new size.
*/
Size
HeapTupleHeader_To73Format(HeapTupleHeader ht, bool compact)
{
HeapTupleHeaderData newdata;
Oid oid;
HeapTupleHeader72 ht72;
int len;

ht72 = (HeapTupleHeader72) ht;
oid = ht72->t_oid;
MemSet(&newdata, 0, sizeof(HeapTupleHeaderData));

/* copy fixed fields */
ItemPointerCopy(&ht72->t_ctid, &newdata.t_ctid);
newdata.t_natts = ht72->t_natts;
newdata.t_infomask = ht72->t_infomask;

HeapTupleHeaderSetXmin(&newdata, ht72->t_xmin);
if (newdata.t_infomask & HEAP_XMAX_INVALID) {
HeapTupleHeaderSetCmin(&newdata, ht72->t_cmin);
}/*if*/
else {
HeapTupleHeaderSetXmax(&newdata, ht72->t_xmax);
}/*else*/

if (newdata.t_infomask & HEAP_MOVED) {
HeapTupleHeaderSetXvac(&newdata, ht72->t_cmin);
}/*if*/
else {
HeapTupleHeaderSetCmax(&newdata, ht72->t_cmax);
}/*else*/

/* move new structure into original position */
len = offsetof(HeapTupleHeaderData, t_bits);
memcpy(ht, &newdata, len);

/* copy bitmap (if there is one) */
if (ht->t_infomask & HEAP_HASNULL) {
int bitmaplen = BITMAPLEN(ht->t_natts);
int off = offsetof(HeapTupleHeader72Data, t_bits);
char *p = (char *) ht;
int i;

Assert(len < off);
for (i = 0; i < bitmaplen; ++i) {
p[len + i] = p[off + i];
}/*for*/

len += bitmaplen;
}/*if*/

/* pad rest with 0 */
Assert(len < ht->t_hoff);
memset((char *)ht + len, 0, ht->t_hoff - len);

/* change length, if requested */
if (compact) {
if (oid != 0) {
len += sizeof(Oid);
}/*if*/

ht->t_hoff = MAXALIGN(len);
}/*if*/

/* copy oid (if there is one) */
if (oid != 0) {
ht->t_infomask |= HEAP_HASOID;
HeapTupleHeaderSetOid(ht, oid);
}/*if*/

return ht->t_hoff;
}

#include <storage/bufpage.h>
#include <access/htup.h>

/*
** Convert a pre 7.3 heap page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
HeapPage_To73Format(Page page, bool compact)
{
PageHeader phdr = (PageHeader)page;
int version = PageGetPageLayoutVersion(page);
Size size = PageGetPageSize(page);
int maxoff = PageGetMaxOffsetNumber(page);
int i;

if (version == PG_PAGE_LAYOUT_VERSION) {
/* already converted */
return;
}/*if*/

Assert(version == 0);

for (i = 1; i <= maxoff; ++i) {
ItemId itid = PageGetItemId(page, i);
// ??? if (ItemIdIsUsed(itid)) ...
HeapTupleHeader ht = PageGetItem(page, itid);
Size oldsz = ht->t_hoff;
Size newsz;

newsz = HeapTupleHeader_To73Format(ht, compact);
if (newsz < oldsz) {
int diff = oldsz - newsz;
ItemOffset off = ItemIdGetOffset(itid);
char *addr;
int lng;
int j;

/* move tuple header to the right */
addr = (char *)ht;
memmove(addr + diff, addr, newsz);
itid->lp_off += diff;
itid->lp_len -= diff;

/*
** Move all tuples that lie to the left of our tuple header.
** (Shamelessly copied from PageIndexTupleDelete()).
*/
addr = (char *) page + phdr->pd_upper;
lng = (int) (off - phdr->pd_upper);
if (lng > 0)
memmove(addr + diff, addr, lng);

memset(addr, 0, diff);

/*
** Adjust upper free space boundary pointer,
** lower is not affected.
*/
phdr->pd_upper += diff;

/* Adjust linp entries. */
for (j = 1; j <= maxoff; ++j) {
ItemId ii = PageGetItemId(page, j);
if (ii->lp_off < off)
ii->lp_off += diff;
}/*for*/
}/*if*/
else
Assert(newsz == oldsz);
}/*for*/

PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
}

/*
** Convert a pre 7.3 page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
Page_To73Format(Page page)
{
int version = PageGetPageLayoutVersion(page);
Size size = PageGetPageSize(page);

if (version == PG_PAGE_LAYOUT_VERSION) {
/* already converted */
return;
}/*if*/

Assert(version == 0);
if (PageGetSpecialSize(page) == 0) {
/*
** Heap page.
** XXX Sure?
** XXX Is there a better way to tell?
*/
HeapPage_To73Format(page, true);
}/*if*/
else {
/*
** Not a heap page: no format change, just adjust version
*/
PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
}/*else*/
}

This should handle all format changes I'm aware of:
. bitmap length
. overlaying fields
. optional oid
. page format version
Am I missing something?

Above code is completely untested, I've not even run it through a
compiler; please consider it as a basis for discussion. If there is
agreement, that we want 7.2 -> 7.3.x pg_upgrade, I'll put more work
into it.

What's missing is mainly a call to Page_To73Format() somewhere. I can
think of
. an input-file-to-output-file-converter run by pg_upgrade instead of
copying/moving the files
. an in-place-converter run by pg_upgrade after copying/moving the
files
. converting each page during normal operation immediately after it
is fetched from disk.

#include <access/htup.h>

typedef struct HeapTupleHeader72Data
{
Oid t_oid; /* OID of this tuple -- 4 bytes */
CommandId t_cmin; /* insert CID stamp -- 4 bytes each */
CommandId t_cmax; /* delete CommandId stamp */
TransactionId t_xmin; /* insert XID stamp -- 4 bytes each */
TransactionId t_xmax; /* delete XID stamp */
ItemPointerData t_ctid; /* current TID of this or newer tuple */
int16 t_natts; /* number of attributes */
uint16 t_infomask; /* various infos */
uint8 t_hoff; /* sizeof() tuple header */
/* ^ - 31 bytes - ^ */
bits8 t_bits[1];
/* bit map of NULLs */
} HeapTupleHeader72Data;

typedef HeapTupleHeader72Data *HeapTupleHeader72;

/*
** Convert a pre-7.3 heap tuple header to 7.3 format.
**
** On entry ht points to a heap tuple header in 7.2 format,
** which will be converted to the new format in place.
** If compact is true, the size of the heap tuple header
** (t_hoff) is reduced, otherwise enough padding bytes are
** inserted to keep the old length.
**
** The return value is the new size.
*/
Size
HeapTupleHeader_To73Format(HeapTupleHeader ht, bool compact)
{
HeapTupleHeaderData newdata;
Oid oid;
HeapTupleHeader72 ht72;
int len;

ht72 = (HeapTupleHeader72) ht;
oid = ht72->t_oid;
MemSet(&newdata, 0, sizeof(HeapTupleHeaderData));

/* copy fixed fields */
ItemPointerCopy(&ht72->t_ctid, &newdata.t_ctid);
newdata.t_natts = ht72->t_natts;
newdata.t_infomask = ht72->t_infomask;

HeapTupleHeaderSetXmin(&newdata, ht72->t_xmin);
if (newdata.t_infomask & HEAP_XMAX_INVALID) {
HeapTupleHeaderSetCmin(&newdata, ht72->t_cmin);
}/*if*/
else {
HeapTupleHeaderSetXmax(&newdata, ht72->t_xmax);
}/*else*/

if (newdata.t_infomask & HEAP_MOVED) {
HeapTupleHeaderSetXvac(&newdata, ht72->t_cmin);
}/*if*/
else {
HeapTupleHeaderSetCmax(&newdata, ht72->t_cmax);
}/*else*/

/* move new structure into original position */
len = offsetof(HeapTupleHeaderData, t_bits);
memcpy(ht, &newdata, len);

/* copy bitmap (if there is one) */
if (ht->t_infomask & HEAP_HASNULL) {
int bitmaplen = BITMAPLEN(ht->t_natts);
int off = offsetof(HeapTupleHeader72Data, t_bits);
char *p = (char *) ht;
int i;

Assert(len < off);
for (i = 0; i < bitmaplen; ++i) {
p[len + i] = p[off + i];
}/*for*/

len += bitmaplen;
}/*if*/

/* pad rest with 0 */
Assert(len < ht->t_hoff);
memset((char *)ht + len, 0, ht->t_hoff - len);

/* change length, if requested */
if (compact) {
if (oid != 0) {
len += sizeof(Oid);
}/*if*/

ht->t_hoff = MAXALIGN(len);
}/*if*/

/* copy oid (if there is one) */
if (oid != 0) {
ht->t_infomask |= HEAP_HASOID;
HeapTupleHeaderSetOid(ht, oid);
}/*if*/

return ht->t_hoff;
}

#include <storage/bufpage.h>
#include <access/htup.h>

/*
** Convert a pre 7.3 heap page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
HeapPage_To73Format(Page page, bool compact)
{
PageHeader phdr = (PageHeader)page;
int version = PageGetPageLayoutVersion(page);
Size size = PageGetPageSize(page);
int maxoff = PageGetMaxOffsetNumber(page);
int i;

if (version == PG_PAGE_LAYOUT_VERSION) {
/* already converted */
return;
}/*if*/

Assert(version == 0);

for (i = 1; i <= maxoff; ++i) {
ItemId itid = PageGetItemId(page, i);
// ??? if (ItemIdIsUsed(itid)) ...
HeapTupleHeader ht = PageGetItem(page, itid);
Size oldsz = ht->t_hoff;
Size newsz;

newsz = HeapTupleHeader_To73Format(ht, compact);
if (newsz < oldsz) {
int diff = oldsz - newsz;
ItemOffset off = ItemIdGetOffset(itid);
char *addr;
int lng;
int j;

/* move tuple header to the right */
addr = (char *)ht;
memmove(addr + diff, addr, newsz);
itid->lp_off += diff;
itid->lp_len -= diff;

/*
** Move all tuples that lie to the left of our tuple header.
** (Shamelessly copied from PageIndexTupleDelete()).
*/
addr = (char *) page + phdr->pd_upper;
lng = (int) (off - phdr->pd_upper);
if (lng > 0)
memmove(addr + diff, addr, lng);

memset(addr, 0, diff);

/*
** Adjust upper free space boundary pointer,
** lower is not affected.
*/
phdr->pd_upper += diff;

/* Adjust linp entries. */
for (j = 1; j <= maxoff; ++j) {
ItemId ii = PageGetItemId(page, j);
if (ii->lp_off < off)
ii->lp_off += diff;
}/*for*/
}/*if*/
else
Assert(newsz == oldsz);
}/*for*/

PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
}

/*
** Convert a pre 7.3 page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
Page_To73Format(Page page)
{
int version = PageGetPageLayoutVersion(page);
Size size = PageGetPageSize(page);

if (version == PG_PAGE_LAYOUT_VERSION) {
/* already converted */
return;
}/*if*/

Assert(version == 0);
if (PageGetSpecialSize(page) == 0) {
/*
** Heap page.
** XXX Sure?
** XXX Is there a better way to tell?
*/
HeapPage_To73Format(page, true);
}/*if*/
else {
/*
** Not a heap page: no format change, just adjust version
*/
PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
}/*else*/
}
Servus
Manfred