Re: Inefficient handling of LO-restore + Patch

Am Donnerstag, 11. April 2002 17:44 schrieb Tom Lane:
> "Mario Weilguni" <mario(dot)weilguni(at)icomedias(dot)com> writes:
> > And I did not find out how I can detect the large object
> > chunksize, either from getting it from the headers (include
> > "storage/large_object.h" did not work)
>

You did not answer if it's ok to post the patch, hope it's ok:
==================================
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.c Mon Feb 11
01:18:20 2002
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.c Thu Apr 11 10:41:09
2002
@@ -819,6 +819,9 @@
AH->createdBlobXref = 1;
}

+ /* Initialize the LO Buffer */
+ AH->lo_buf_used = 0;
+
/*
* Start long-running TXs if necessary
*/
@@ -848,6 +851,19 @@
void
EndRestoreBlob(ArchiveHandle *AH, Oid oid)
{
+ if(AH->lo_buf_used > 0) {
+ /* Write remaining bytes from the LO buffer */
+ int res;
+ res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf,
AH->lo_buf_used);
+
+ ahlog(AH, 5, "wrote remaining %d bytes of large object data (result =
%d)\n",
+ (int)AH->lo_buf_used, res);
+ if (res != AH->lo_buf_used)
+ die_horribly(AH, modulename, "could not write to large object (result: %d,
expected: %d)\n",
+ res, AH->lo_buf_used);
+ AH->lo_buf_used = 0;
+ }
+
lo_close(AH->connection, AH->loFd);
AH->writingBlob = 0;

@@ -1228,14 +1244,27 @@

if (AH->writingBlob)
{
- res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb);
- ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
- (int) (size * nmemb), res);
- if (res != size * nmemb)
+ if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) {
+ /* Split LO buffer */
+ int remaining = AH->lo_buf_size - AH->lo_buf_used;
+ int slack = nmemb * size - remaining;
+
+ memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining);
+ res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size);
+ ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
+ AH->lo_buf_size, res);
+ if (res != AH->lo_buf_size)
die_horribly(AH, modulename, "could not write to large object (result: %d,
expected: %d)\n",
- res, (int) (size * nmemb));
+ res, AH->lo_buf_size);
+ memcpy(AH->lo_buf, ptr + remaining, slack);
+ AH->lo_buf_used = slack;
+ } else {
+ /* LO Buffer is still large enough, buffer it */
+ memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb);
+ AH->lo_buf_used += size * nmemb;
+ }

- return res;
+ return size * nmemb;
}
else if (AH->gzOut)
{
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.h
postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.h
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.h Mon Nov 5
18:46:30 2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.h Thu Apr 11 10:41:14
2002
@@ -41,6 +41,7 @@
#include <errno.h>

#include "pqexpbuffer.h"
+#define LOBBUFSIZE 32768

#ifdef HAVE_LIBZ
#include <zlib.h>
@@ -240,6 +241,9 @@

RestoreOptions *ropt; /* Used to check restore options in
* ahwrite etc */
+ void *lo_buf;
+ int lo_buf_used;
+ int lo_buf_size;
} ArchiveHandle;

typedef struct _tocEntry
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_custom.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_custom.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_custom.c Wed Nov 28
00:48:12 2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_custom.c Thu Apr 11 10:42:45
2002
@@ -153,6 +153,12 @@
if (ctx->zp == NULL)
die_horribly(AH, modulename, "out of memory\n");

+ /* Initialize LO buffering */
+ AH->lo_buf_size = LOBBUFSIZE;
+ AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+ if(AH->lo_buf == NULL)
+ die_horribly(AH, modulename, "out of memory\n");
+
/*
* zlibOutSize is the buffer size we tell zlib it can output to. We
* actually allocate one extra byte because some routines want to
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_files.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_files.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_files.c Thu Oct 25
07:49:52 2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_files.c Thu Apr 11 10:43:01
2002
@@ -113,6 +113,12 @@
AH->formatData = (void *) ctx;
ctx->filePos = 0;

+ /* Initialize LO buffering */
+ AH->lo_buf_size = LOBBUFSIZE;
+ AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+ if(AH->lo_buf == NULL)
+ die_horribly(AH, modulename, "out of memory\n");
+
/*
* Now open the TOC file
*/
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_null.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_null.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_null.c Wed Jun 27 23:21:37
2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_null.c Thu Apr 11 10:44:53 2002
@@ -64,7 +64,6 @@
*/
if (AH->mode == archModeRead)
die_horribly(AH, NULL, "this format cannot be read\n");
-
}

/*
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_tar.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_tar.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_tar.c Sun Oct 28 07:25:58
2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_tar.c Thu Apr 11 10:44:08 2002
@@ -157,6 +157,12 @@
ctx = (lclContext *) malloc(sizeof(lclContext));
AH->formatData = (void *) ctx;
ctx->filePos = 0;
+
+ /* Initialize LO buffering */
+ AH->lo_buf_size = LOBBUFSIZE;
+ AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+ if(AH->lo_buf == NULL)
+ die_horribly(AH, modulename, "out of memory\n");

/*
* Now open the TOC file
============================

>
> ---------------------------(end of broadcast)---------------------------
> TIP 4: Don't 'kill -9' the postmaster