Skip site navigation (1) Skip section navigation (2)

Re: Upgrading rant.

From: Manfred Koizar <mkoi-pg(at)aon(dot)at>
To: Tom Lane <tgl(at)sss(dot)pgh(dot)pa(dot)us>
Cc: Hannu Krosing <hannu(at)tm(dot)ee>, mlw <pgsql(at)mohawksoft(dot)com>,Lamar Owen <lamar(dot)owen(at)wgcr(dot)org>, pgsql-hackers(at)postgresql(dot)org
Subject: Re: Upgrading rant.
Date: 2003-01-07 10:18:15
Message-ID: jq7l1vgiqv6no36u0855hub66imubcg1l5@4ax.com (view raw or flat)
Thread:
Lists: pgsql-hackers
On Fri, 03 Jan 2003 15:37:56 -0500, Tom Lane <tgl(at)sss(dot)pgh(dot)pa(dot)us> wrote:
>The system tables are not the problem. [...]
>
>Changes in the on-disk representation of user tables would be harder to
>deal with, but they are also much rarer (AFAIR we've only done that
>twice: WAL required additions to page and tuple headers, and then there
>were Manfred's space-saving changes in 7.3).

So I'm the bad guy? ;-)  AFAICS handling the page and tuple format
changes doesn't need much more than what I have hacked together
yesterday afternoon:

#include <access/htup.h>

typedef struct HeapTupleHeader72Data
{
	Oid			t_oid;			/* OID of this tuple -- 4 bytes */
	CommandId	t_cmin;			/* insert CID stamp -- 4 bytes each */
	CommandId	t_cmax;			/* delete CommandId stamp */
	TransactionId t_xmin;		/* insert XID stamp -- 4 bytes each */
	TransactionId t_xmax;		/* delete XID stamp */
	ItemPointerData t_ctid;		/* current TID of this or newer tuple */
	int16		t_natts;		/* number of attributes */
	uint16		t_infomask;		/* various infos */
	uint8		t_hoff;			/* sizeof() tuple header */
	/* ^ - 31 bytes - ^ */
	bits8		t_bits[1];
	/* bit map of NULLs */
} HeapTupleHeader72Data;

typedef HeapTupleHeader72Data *HeapTupleHeader72;

/*
** Convert a pre-7.3 heap tuple header to 7.3 format.
**
** On entry ht points to a heap tuple header in 7.2 format,
** which will be converted to the new format in place.
** If compact is true, the size of the heap tuple header
** (t_hoff) is reduced, otherwise enough padding bytes are
** inserted to keep the old length.
**
** The return value is the new size.
*/
Size
HeapTupleHeader_To73Format(HeapTupleHeader ht, bool compact)
{
	HeapTupleHeaderData newdata;
	Oid oid;
	HeapTupleHeader72 ht72;
	int len;

	ht72 = (HeapTupleHeader72) ht;
	oid = ht72->t_oid;
	MemSet(&newdata, 0, sizeof(HeapTupleHeaderData));

	/* copy fixed fields */
	ItemPointerCopy(&ht72->t_ctid, &newdata.t_ctid);
	newdata.t_natts = ht72->t_natts;
	newdata.t_infomask = ht72->t_infomask;

	HeapTupleHeaderSetXmin(&newdata, ht72->t_xmin);
	if (newdata.t_infomask & HEAP_XMAX_INVALID) {
		HeapTupleHeaderSetCmin(&newdata, ht72->t_cmin);
	}/*if*/
	else {
		HeapTupleHeaderSetXmax(&newdata, ht72->t_xmax);
	}/*else*/

	if (newdata.t_infomask & HEAP_MOVED) {
		HeapTupleHeaderSetXvac(&newdata, ht72->t_cmin);
	}/*if*/
	else {
		HeapTupleHeaderSetCmax(&newdata, ht72->t_cmax);
	}/*else*/

	/* move new structure into original position */
	len = offsetof(HeapTupleHeaderData, t_bits);
	memcpy(ht, &newdata, len);

	/* copy bitmap (if there is one) */
	if (ht->t_infomask & HEAP_HASNULL) {
		int bitmaplen = BITMAPLEN(ht->t_natts);
		int off = offsetof(HeapTupleHeader72Data, t_bits);
		char *p = (char *) ht;
		int i;

		Assert(len < off);
		for (i = 0; i < bitmaplen; ++i) {
			p[len + i] = p[off + i];
		}/*for*/

		len += bitmaplen;
	}/*if*/

	/* pad rest with 0 */
	Assert(len < ht->t_hoff);
	memset((char *)ht + len, 0, ht->t_hoff - len);

	/* change length, if requested */
	if (compact) {
		if (oid != 0) {
			len += sizeof(Oid);
		}/*if*/

		ht->t_hoff = MAXALIGN(len);
	}/*if*/

	/* copy oid (if there is one) */
	if (oid != 0) {
		ht->t_infomask |= HEAP_HASOID;
		HeapTupleHeaderSetOid(ht, oid);
	}/*if*/

	return ht->t_hoff;
}

#include <storage/bufpage.h>
#include <access/htup.h>

/*
** Convert a pre 7.3 heap page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
HeapPage_To73Format(Page page, bool compact)
{
	PageHeader phdr = (PageHeader)page;
	int version = PageGetPageLayoutVersion(page);
	Size size = PageGetPageSize(page);
	int maxoff = PageGetMaxOffsetNumber(page);
	int i;

	if (version == PG_PAGE_LAYOUT_VERSION) {
		/* already converted */
		return;
	}/*if*/

	Assert(version == 0);

	for (i = 1; i <= maxoff; ++i) {
		ItemId itid = PageGetItemId(page, i);
		// ??? if (ItemIdIsUsed(itid)) ...
		HeapTupleHeader ht = PageGetItem(page, itid);
		Size oldsz = ht->t_hoff;
		Size newsz;

		newsz = HeapTupleHeader_To73Format(ht, compact);
		if (newsz < oldsz) {
			int diff = oldsz - newsz;
			ItemOffset off = ItemIdGetOffset(itid);
			char *addr;
			int lng;
			int j;

			/* move tuple header to the right */
			addr = (char *)ht;
			memmove(addr + diff, addr, newsz);
			itid->lp_off += diff;
			itid->lp_len -= diff;

			/*
			** Move all tuples that lie to the left of our tuple header.
			** (Shamelessly copied from PageIndexTupleDelete()).
			*/
			addr = (char *) page + phdr->pd_upper;
			lng = (int) (off - phdr->pd_upper);
			if (lng > 0)
				memmove(addr + diff, addr, lng);

			memset(addr, 0, diff);

			/*
			** Adjust upper free space boundary pointer,
			** lower is not affected.
			*/
			phdr->pd_upper += diff;

			/* Adjust linp entries. */
			for (j = 1; j <= maxoff; ++j) {
				ItemId ii = PageGetItemId(page, j);
				if (ii->lp_off < off)
					ii->lp_off += diff;
			}/*for*/
		}/*if*/
		else
			Assert(newsz == oldsz);
	}/*for*/

	PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
}

/*
** Convert a pre 7.3 page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
Page_To73Format(Page page)
{
	int version = PageGetPageLayoutVersion(page);
	Size size = PageGetPageSize(page);

	if (version == PG_PAGE_LAYOUT_VERSION) {
		/* already converted */
		return;
	}/*if*/

	Assert(version == 0);
	if (PageGetSpecialSize(page) == 0) {
		/*
		** Heap page.
		** XXX Sure?
		** XXX Is there a better way to tell?
		*/
		HeapPage_To73Format(page, true);
	}/*if*/
	else {
		/*
		** Not a heap page: no format change, just adjust version
		*/
		PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
	}/*else*/
}

This should handle all format changes I'm aware of:
 .  bitmap length
 .  overlaying fields
 .  optional oid
 .  page format version
Am I missing something?

Above code is completely untested, I've not even run it through a
compiler; please consider it as a basis for discussion.  If there is
agreement, that we want 7.2 -> 7.3.x pg_upgrade, I'll put more work
into it.

What's missing is mainly a call to Page_To73Format() somewhere.  I can
think of
 .  an input-file-to-output-file-converter run by pg_upgrade instead of
    copying/moving the files
 .  an in-place-converter run by pg_upgrade after copying/moving the
    files
 .  converting each page during normal operation immediately after it
    is fetched from disk.

#include <access/htup.h>

typedef struct HeapTupleHeader72Data
{
	Oid			t_oid;			/* OID of this tuple -- 4 bytes */
	CommandId	t_cmin;			/* insert CID stamp -- 4 bytes each */
	CommandId	t_cmax;			/* delete CommandId stamp */
	TransactionId t_xmin;		/* insert XID stamp -- 4 bytes each */
	TransactionId t_xmax;		/* delete XID stamp */
	ItemPointerData t_ctid;		/* current TID of this or newer tuple */
	int16		t_natts;		/* number of attributes */
	uint16		t_infomask;		/* various infos */
	uint8		t_hoff;			/* sizeof() tuple header */
	/* ^ - 31 bytes - ^ */
	bits8		t_bits[1];
	/* bit map of NULLs */
} HeapTupleHeader72Data;

typedef HeapTupleHeader72Data *HeapTupleHeader72;

/*
** Convert a pre-7.3 heap tuple header to 7.3 format.
**
** On entry ht points to a heap tuple header in 7.2 format,
** which will be converted to the new format in place.
** If compact is true, the size of the heap tuple header
** (t_hoff) is reduced, otherwise enough padding bytes are
** inserted to keep the old length.
**
** The return value is the new size.
*/
Size
HeapTupleHeader_To73Format(HeapTupleHeader ht, bool compact)
{
	HeapTupleHeaderData newdata;
	Oid oid;
	HeapTupleHeader72 ht72;
	int len;

	ht72 = (HeapTupleHeader72) ht;
	oid = ht72->t_oid;
	MemSet(&newdata, 0, sizeof(HeapTupleHeaderData));

	/* copy fixed fields */
	ItemPointerCopy(&ht72->t_ctid, &newdata.t_ctid);
	newdata.t_natts = ht72->t_natts;
	newdata.t_infomask = ht72->t_infomask;

	HeapTupleHeaderSetXmin(&newdata, ht72->t_xmin);
	if (newdata.t_infomask & HEAP_XMAX_INVALID) {
		HeapTupleHeaderSetCmin(&newdata, ht72->t_cmin);
	}/*if*/
	else {
		HeapTupleHeaderSetXmax(&newdata, ht72->t_xmax);
	}/*else*/

	if (newdata.t_infomask & HEAP_MOVED) {
		HeapTupleHeaderSetXvac(&newdata, ht72->t_cmin);
	}/*if*/
	else {
		HeapTupleHeaderSetCmax(&newdata, ht72->t_cmax);
	}/*else*/

	/* move new structure into original position */
	len = offsetof(HeapTupleHeaderData, t_bits);
	memcpy(ht, &newdata, len);

	/* copy bitmap (if there is one) */
	if (ht->t_infomask & HEAP_HASNULL) {
		int bitmaplen = BITMAPLEN(ht->t_natts);
		int off = offsetof(HeapTupleHeader72Data, t_bits);
		char *p = (char *) ht;
		int i;

		Assert(len < off);
		for (i = 0; i < bitmaplen; ++i) {
			p[len + i] = p[off + i];
		}/*for*/

		len += bitmaplen;
	}/*if*/

	/* pad rest with 0 */
	Assert(len < ht->t_hoff);
	memset((char *)ht + len, 0, ht->t_hoff - len);

	/* change length, if requested */
	if (compact) {
		if (oid != 0) {
			len += sizeof(Oid);
		}/*if*/

		ht->t_hoff = MAXALIGN(len);
	}/*if*/

	/* copy oid (if there is one) */
	if (oid != 0) {
		ht->t_infomask |= HEAP_HASOID;
		HeapTupleHeaderSetOid(ht, oid);
	}/*if*/

	return ht->t_hoff;
}

#include <storage/bufpage.h>
#include <access/htup.h>

/*
** Convert a pre 7.3 heap page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
HeapPage_To73Format(Page page, bool compact)
{
	PageHeader phdr = (PageHeader)page;
	int version = PageGetPageLayoutVersion(page);
	Size size = PageGetPageSize(page);
	int maxoff = PageGetMaxOffsetNumber(page);
	int i;

	if (version == PG_PAGE_LAYOUT_VERSION) {
		/* already converted */
		return;
	}/*if*/

	Assert(version == 0);

	for (i = 1; i <= maxoff; ++i) {
		ItemId itid = PageGetItemId(page, i);
		// ??? if (ItemIdIsUsed(itid)) ...
		HeapTupleHeader ht = PageGetItem(page, itid);
		Size oldsz = ht->t_hoff;
		Size newsz;

		newsz = HeapTupleHeader_To73Format(ht, compact);
		if (newsz < oldsz) {
			int diff = oldsz - newsz;
			ItemOffset off = ItemIdGetOffset(itid);
			char *addr;
			int lng;
			int j;

			/* move tuple header to the right */
			addr = (char *)ht;
			memmove(addr + diff, addr, newsz);
			itid->lp_off += diff;
			itid->lp_len -= diff;

			/*
			** Move all tuples that lie to the left of our tuple header.
			** (Shamelessly copied from PageIndexTupleDelete()).
			*/
			addr = (char *) page + phdr->pd_upper;
			lng = (int) (off - phdr->pd_upper);
			if (lng > 0)
				memmove(addr + diff, addr, lng);

			memset(addr, 0, diff);

			/*
			** Adjust upper free space boundary pointer,
			** lower is not affected.
			*/
			phdr->pd_upper += diff;

			/* Adjust linp entries. */
			for (j = 1; j <= maxoff; ++j) {
				ItemId ii = PageGetItemId(page, j);
				if (ii->lp_off < off)
					ii->lp_off += diff;
			}/*for*/
		}/*if*/
		else
			Assert(newsz == oldsz);
	}/*for*/

	PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
}

/*
** Convert a pre 7.3 page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
Page_To73Format(Page page)
{
	int version = PageGetPageLayoutVersion(page);
	Size size = PageGetPageSize(page);

	if (version == PG_PAGE_LAYOUT_VERSION) {
		/* already converted */
		return;
	}/*if*/

	Assert(version == 0);
	if (PageGetSpecialSize(page) == 0) {
		/*
		** Heap page.
		** XXX Sure?
		** XXX Is there a better way to tell?
		*/
		HeapPage_To73Format(page, true);
	}/*if*/
	else {
		/*
		** Not a heap page: no format change, just adjust version
		*/
		PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
	}/*else*/
}
Servus
 Manfred

In response to

Responses

pgsql-hackers by date

Next:From: Manfred KoizarDate: 2003-01-07 10:39:58
Subject: Re: Upgrading rant.
Previous:From: Dave PageDate: 2003-01-07 10:06:08
Subject: Re: [HACKERS] Thank-you to Cybertec Geschwinde & Schonig

Privacy Policy | About PostgreSQL
Copyright © 1996-2014 The PostgreSQL Global Development Group