Re: Inefficient handling of LO-restore + Patch

Your patch has been added to the PostgreSQL unapplied patches list at:

http://candle.pha.pa.us/cgi-bin/pgpatches

I will try to apply it within the next 48 hours.

---------------------------------------------------------------------------

Mario Weilguni wrote:
> Am Donnerstag, 11. April 2002 17:44 schrieb Tom Lane:
> > "Mario Weilguni" <mario(dot)weilguni(at)icomedias(dot)com> writes:
> > > And I did not find out how I can detect the large object
> > > chunksize, either from getting it from the headers (include
> > > "storage/large_object.h" did not work)
> >
>
> You did not answer if it's ok to post the patch, hope it's ok:
> ==================================
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.c
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.c Mon Feb 11
> 01:18:20 2002
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.c Thu Apr 11 10:41:09
> 2002
> @@ -819,6 +819,9 @@
> AH->createdBlobXref = 1;
> }
>
> + /* Initialize the LO Buffer */
> + AH->lo_buf_used = 0;
> +
> /*
> * Start long-running TXs if necessary
> */
> @@ -848,6 +851,19 @@
> void
> EndRestoreBlob(ArchiveHandle *AH, Oid oid)
> {
> + if(AH->lo_buf_used > 0) {
> + /* Write remaining bytes from the LO buffer */
> + int res;
> + res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf,
> AH->lo_buf_used);
> +
> + ahlog(AH, 5, "wrote remaining %d bytes of large object data (result =
> %d)\n",
> + (int)AH->lo_buf_used, res);
> + if (res != AH->lo_buf_used)
> + die_horribly(AH, modulename, "could not write to large object (result: %d,
> expected: %d)\n",
> + res, AH->lo_buf_used);
> + AH->lo_buf_used = 0;
> + }
> +
> lo_close(AH->connection, AH->loFd);
> AH->writingBlob = 0;
>
> @@ -1228,14 +1244,27 @@
>
> if (AH->writingBlob)
> {
> - res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb);
> - ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
> - (int) (size * nmemb), res);
> - if (res != size * nmemb)
> + if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) {
> + /* Split LO buffer */
> + int remaining = AH->lo_buf_size - AH->lo_buf_used;
> + int slack = nmemb * size - remaining;
> +
> + memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining);
> + res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size);
> + ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
> + AH->lo_buf_size, res);
> + if (res != AH->lo_buf_size)
> die_horribly(AH, modulename, "could not write to large object (result: %d,
> expected: %d)\n",
> - res, (int) (size * nmemb));
> + res, AH->lo_buf_size);
> + memcpy(AH->lo_buf, ptr + remaining, slack);
> + AH->lo_buf_used = slack;
> + } else {
> + /* LO Buffer is still large enough, buffer it */
> + memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb);
> + AH->lo_buf_used += size * nmemb;
> + }
>
> - return res;
> + return size * nmemb;
> }
> else if (AH->gzOut)
> {
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.h
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.h
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.h Mon Nov 5
> 18:46:30 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.h Thu Apr 11 10:41:14
> 2002
> @@ -41,6 +41,7 @@
> #include <errno.h>
>
> #include "pqexpbuffer.h"
> +#define LOBBUFSIZE 32768
>
> #ifdef HAVE_LIBZ
> #include <zlib.h>
> @@ -240,6 +241,9 @@
>
> RestoreOptions *ropt; /* Used to check restore options in
> * ahwrite etc */
> + void *lo_buf;
> + int lo_buf_used;
> + int lo_buf_size;
> } ArchiveHandle;
>
> typedef struct _tocEntry
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_custom.c
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_custom.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_custom.c Wed Nov 28
> 00:48:12 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_custom.c Thu Apr 11 10:42:45
> 2002
> @@ -153,6 +153,12 @@
> if (ctx->zp == NULL)
> die_horribly(AH, modulename, "out of memory\n");
>
> + /* Initialize LO buffering */
> + AH->lo_buf_size = LOBBUFSIZE;
> + AH->lo_buf = (void *)malloc(LOBBUFSIZE);
> + if(AH->lo_buf == NULL)
> + die_horribly(AH, modulename, "out of memory\n");
> +
> /*
> * zlibOutSize is the buffer size we tell zlib it can output to. We
> * actually allocate one extra byte because some routines want to
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_files.c
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_files.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_files.c Thu Oct 25
> 07:49:52 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_files.c Thu Apr 11 10:43:01
> 2002
> @@ -113,6 +113,12 @@
> AH->formatData = (void *) ctx;
> ctx->filePos = 0;
>
> + /* Initialize LO buffering */
> + AH->lo_buf_size = LOBBUFSIZE;
> + AH->lo_buf = (void *)malloc(LOBBUFSIZE);
> + if(AH->lo_buf == NULL)
> + die_horribly(AH, modulename, "out of memory\n");
> +
> /*
> * Now open the TOC file
> */
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_null.c
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_null.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_null.c Wed Jun 27 23:21:37
> 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_null.c Thu Apr 11 10:44:53 2002
> @@ -64,7 +64,6 @@
> */
> if (AH->mode == archModeRead)
> die_horribly(AH, NULL, "this format cannot be read\n");
> -
> }
>
> /*
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_tar.c
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_tar.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_tar.c Sun Oct 28 07:25:58
> 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_tar.c Thu Apr 11 10:44:08 2002
> @@ -157,6 +157,12 @@
> ctx = (lclContext *) malloc(sizeof(lclContext));
> AH->formatData = (void *) ctx;
> ctx->filePos = 0;
> +
> + /* Initialize LO buffering */
> + AH->lo_buf_size = LOBBUFSIZE;
> + AH->lo_buf = (void *)malloc(LOBBUFSIZE);
> + if(AH->lo_buf == NULL)
> + die_horribly(AH, modulename, "out of memory\n");
>
> /*
> * Now open the TOC file
> ============================
>
> >
> > ---------------------------(end of broadcast)---------------------------
> > TIP 4: Don't 'kill -9' the postmaster
>
> ---------------------------(end of broadcast)---------------------------
> TIP 6: Have you searched our list archives?
>
> http://archives.postgresql.org
>

--
Bruce Momjian | http://candle.pha.pa.us
pgman(at)candle(dot)pha(dot)pa(dot)us | (610) 853-3000
+ If your life is a hard drive, | 830 Blythe Avenue
+ Christ can be your backup. | Drexel Hill, Pennsylvania 19026