From 98d21da4d10c558323cef1f3895f02b3088345ed Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Thu, 29 Jan 2015 11:41:35 +0100 Subject: [PATCH 2/3] copydir LSN v2 --- src/backend/commands/dbcommands.c | 32 ++++++++++--------- src/backend/storage/file/copydir.c | 64 +++++++++++++++++++++++++++++++++++--- src/backend/storage/file/reinit.c | 3 +- src/include/storage/copydir.h | 6 ++-- 4 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 5e66961..6dd9878 100644 *** a/src/backend/commands/dbcommands.c --- b/src/backend/commands/dbcommands.c *************** createdb(const CreatedbStmt *stmt) *** 586,591 **** --- 586,592 ---- Oid dsttablespace; char *srcpath; char *dstpath; + XLogRecPtr recptr; struct stat st; /* No need to copy global tablespace */ *************** createdb(const CreatedbStmt *stmt) *** 609,621 **** dstpath = GetDatabasePath(dboid, dsttablespace); - /* - * Copy this subdirectory to the new location - * - * We don't need to copy subdirectories - */ - copydir(srcpath, dstpath, false); - /* Record the filesystem change in XLOG */ { xl_dbase_create_rec xlrec; --- 610,615 ---- *************** createdb(const CreatedbStmt *stmt) *** 628,636 **** XLogBeginInsert(); XLogRegisterData((char *) &xlrec, sizeof(xl_dbase_create_rec)); ! (void) XLogInsert(RM_DBASE_ID, XLOG_DBASE_CREATE | XLR_SPECIAL_REL_UPDATE); } } heap_endscan(scan); heap_close(rel, AccessShareLock); --- 622,637 ---- XLogBeginInsert(); XLogRegisterData((char *) &xlrec, sizeof(xl_dbase_create_rec)); ! recptr = XLogInsert(RM_DBASE_ID, XLOG_DBASE_CREATE | XLR_SPECIAL_REL_UPDATE); } + + /* + * Copy this subdirectory to the new location + * + * We don't need to copy subdirectories + */ + copydir(srcpath, dstpath, false, recptr); } heap_endscan(scan); heap_close(rel, AccessShareLock); *************** movedb(const char *dbname, const char *t *** 1214,1223 **** PG_ENSURE_ERROR_CLEANUP(movedb_failure_callback, PointerGetDatum(&fparms)); { ! /* ! * Copy files from the old tablespace to the new one ! */ ! copydir(src_dbpath, dst_dbpath, false); /* * Record the filesystem change in XLOG --- 1215,1221 ---- PG_ENSURE_ERROR_CLEANUP(movedb_failure_callback, PointerGetDatum(&fparms)); { ! XLogRecPtr recptr; /* * Record the filesystem change in XLOG *************** movedb(const char *dbname, const char *t *** 1233,1243 **** XLogBeginInsert(); XLogRegisterData((char *) &xlrec, sizeof(xl_dbase_create_rec)); ! (void) XLogInsert(RM_DBASE_ID, XLOG_DBASE_CREATE | XLR_SPECIAL_REL_UPDATE); } /* * Update the database's pg_database tuple */ ScanKeyInit(&scankey, --- 1231,1246 ---- XLogBeginInsert(); XLogRegisterData((char *) &xlrec, sizeof(xl_dbase_create_rec)); ! recptr = XLogInsert(RM_DBASE_ID, XLOG_DBASE_CREATE | XLR_SPECIAL_REL_UPDATE); } /* + * Copy files from the old tablespace to the new one + */ + copydir(src_dbpath, dst_dbpath, false, recptr); + + /* * Update the database's pg_database tuple */ ScanKeyInit(&scankey, *************** dbase_redo(XLogReaderState *record) *** 2045,2050 **** --- 2048,2054 ---- if (info == XLOG_DBASE_CREATE) { xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record); + XLogRecPtr lsn = record->EndRecPtr; char *src_path; char *dst_path; struct stat st; *************** dbase_redo(XLogReaderState *record) *** 2077,2083 **** * * We don't need to copy subdirectories */ ! copydir(src_path, dst_path, false); } else if (info == XLOG_DBASE_DROP) { --- 2081,2087 ---- * * We don't need to copy subdirectories */ ! copydir(src_path, dst_path, false, lsn); } else if (info == XLOG_DBASE_DROP) { diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c index 41b2c62..a7a0dc5 100644 *** a/src/backend/storage/file/copydir.c --- b/src/backend/storage/file/copydir.c *************** *** 22,27 **** --- 22,29 ---- #include #include + #include "common/relpath.h" + #include "storage/bufpage.h" #include "storage/copydir.h" #include "storage/fd.h" #include "miscadmin.h" *************** *** 32,40 **** * * If recurse is false, subdirectories are ignored. Anything that's not * a directory or a regular file is ignored. */ void ! copydir(char *fromdir, char *todir, bool recurse) { DIR *xldir; struct dirent *xlde; --- 34,45 ---- * * If recurse is false, subdirectories are ignored. Anything that's not * a directory or a regular file is ignored. + * + * If recptr is different from InvalidXlogRecPtr, LSN of pages in the + * destination directory will be updated to recptr. */ void ! copydir(char *fromdir, char *todir, bool recurse, XLogRecPtr recptr) { DIR *xldir; struct dirent *xlde; *************** copydir(char *fromdir, char *todir, bool *** 75,84 **** { /* recurse to handle subdirectories */ if (recurse) ! copydir(fromfile, tofile, true); } else if (S_ISREG(fst.st_mode)) ! copy_file(fromfile, tofile); } FreeDir(xldir); --- 80,106 ---- { /* recurse to handle subdirectories */ if (recurse) ! copydir(fromfile, tofile, true, recptr); } else if (S_ISREG(fst.st_mode)) ! { ! int oidchars; ! ForkNumber fork; ! ! /* ! * To support incremental backups, we need to update the LSN in ! * all relation files we are copying. ! * ! * We are updating only the MAIN fork because at the moment ! * blocks in FSM and VM forks are not guaranteed to have an ! * up-to-date LSN ! */ ! if (parse_filename_for_nontemp_relation(xlde->d_name, ! &oidchars, &fork) && fork == MAIN_FORKNUM) ! copy_file(fromfile, tofile, recptr); ! else ! copy_file(fromfile, tofile, InvalidXLogRecPtr); ! } } FreeDir(xldir); *************** copydir(char *fromdir, char *todir, bool *** 130,144 **** /* * copy one file */ void ! copy_file(char *fromfile, char *tofile) { char *buffer; int srcfd; int dstfd; int nbytes; off_t offset; /* Use palloc to ensure we get a maxaligned buffer */ #define COPY_BUF_SIZE (8 * BLCKSZ) --- 152,170 ---- /* * copy one file + * + * If recptr is different from InvalidXlogRecPtr, the destination file will + * have all its pages with LSN set accordingly */ void ! copy_file(char *fromfile, char *tofile, XLogRecPtr recptr) { char *buffer; int srcfd; int dstfd; int nbytes; off_t offset; + BlockNumber blkno = 0; /* Use palloc to ensure we get a maxaligned buffer */ #define COPY_BUF_SIZE (8 * BLCKSZ) *************** copy_file(char *fromfile, char *tofile) *** 176,181 **** --- 202,237 ---- errmsg("could not read file \"%s\": %m", fromfile))); if (nbytes == 0) break; + + /* + * If a valid recptr has been provided, the resulting file will have + * all its pages with LSN set accordingly + */ + if (recptr != InvalidXLogRecPtr) + { + char *page; + + /* + * If we are updating LSN of a file, we must be sure that the + * source file is not being extended. + */ + if (nbytes % BLCKSZ != 0) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("file \"%s\" size is not multiple of %d", + fromfile, BLCKSZ))); + + for (page = buffer; page < (buffer + nbytes); page += BLCKSZ, blkno++) + { + /* Update LSN only if the page looks valid */ + if (!PageIsNew(page) && PageIsVerified(page, blkno)) + { + PageSetLSN(page, recptr); + PageSetChecksumInplace(page, blkno); + } + } + } + errno = 0; if ((int) write(dstfd, buffer, nbytes) != nbytes) { diff --git a/src/backend/storage/file/reinit.c b/src/backend/storage/file/reinit.c index 02b5fee..854ae4a 100644 *** a/src/backend/storage/file/reinit.c --- b/src/backend/storage/file/reinit.c *************** *** 16,21 **** --- 16,22 ---- #include + #include "access/xlogdefs.h" #include "catalog/catalog.h" #include "common/relpath.h" #include "storage/copydir.h" *************** ResetUnloggedRelationsInDbspaceDir(const *** 333,339 **** /* OK, we're ready to perform the actual copy. */ elog(DEBUG2, "copying %s to %s", srcpath, dstpath); ! copy_file(srcpath, dstpath); } FreeDir(dbspace_dir); --- 334,340 ---- /* OK, we're ready to perform the actual copy. */ elog(DEBUG2, "copying %s to %s", srcpath, dstpath); ! copy_file(srcpath, dstpath, InvalidXLogRecPtr); } FreeDir(dbspace_dir); diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 2635a7e..463141d 100644 *** a/src/include/storage/copydir.h --- b/src/include/storage/copydir.h *************** *** 13,19 **** #ifndef COPYDIR_H #define COPYDIR_H ! extern void copydir(char *fromdir, char *todir, bool recurse); ! extern void copy_file(char *fromfile, char *tofile); #endif /* COPYDIR_H */ --- 13,21 ---- #ifndef COPYDIR_H #define COPYDIR_H ! #include "access/xlogdefs.h" ! ! extern void copydir(char *fromdir, char *todir, bool recurse, XLogRecPtr recptr); ! extern void copy_file(char *fromfile, char *tofile, XLogRecPtr recptr); #endif /* COPYDIR_H */ -- 2.2.2