From 21bf196a5b36c66432d86d84c2f1027ace2c1838 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <bossartn@amazon.com>
Date: Sun, 24 Oct 2021 04:40:15 +0000
Subject: [PATCH v5 2/2] Add an example of a basic archiving module.

---
 contrib/Makefile                      |   1 +
 contrib/basic_archive/Makefile        |  15 ++
 contrib/basic_archive/basic_archive.c | 385 ++++++++++++++++++++++++++++++++++
 3 files changed, 401 insertions(+)
 create mode 100644 contrib/basic_archive/Makefile
 create mode 100644 contrib/basic_archive/basic_archive.c

diff --git a/contrib/Makefile b/contrib/Makefile
index f27e458482..aff834ebaa 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -9,6 +9,7 @@ SUBDIRS = \
 		amcheck		\
 		auth_delay	\
 		auto_explain	\
+		basic_archive	\
 		bloom		\
 		btree_gin	\
 		btree_gist	\
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
new file mode 100644
index 0000000000..ea6b460889
--- /dev/null
+++ b/contrib/basic_archive/Makefile
@@ -0,0 +1,15 @@
+# contrib/basic_archive/Makefile
+
+MODULES = basic_archive
+PGFILEDESC = "basic_archive - basic archive module"
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/basic_archive
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
new file mode 100644
index 0000000000..6c8ffa0520
--- /dev/null
+++ b/contrib/basic_archive/basic_archive.c
@@ -0,0 +1,385 @@
+/*-------------------------------------------------------------------------
+ *
+ * basic_archive.c
+ *
+ * This file demonstrates a basic archive library implementation that is
+ * roughly equivalent to the following shell command:
+ *
+ * 		test ! -f /path/to/dest && cp /path/to/src /path/to/dest
+ *
+ * One notable difference between this module and the shell command above
+ * is that this module first copies the file to a temporary destination,
+ * syncs it to disk, and then durably moves it to the final destination.
+ * While this module is designed to gracefully recover from inconvenient
+ * server crashes (e.g., a crash after we've archived the file but before
+ * we've renamed its .ready file to .done), it does not have any special
+ * handling for multiple servers archiving to the same location.
+ *
+ * Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  contrib/basic_archive/basic_archive.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "common/checksum_helper.h"
+#include "fmgr.h"
+#include "miscadmin.h"
+#include "postmaster/pgarch.h"
+#include "storage/fd.h"
+#include "utils/guc.h"
+
+PG_MODULE_MAGIC;
+
+void _PG_archive_module_init(ArchiveModuleCallbacks *cb);
+
+static char *archive_directory = NULL;
+
+static bool basic_archive_configured(void);
+static bool basic_archive_file(const char *file, const char *path);
+static bool check_archive_directory(char **newval, void **extra, GucSource source);
+static void copy_file(const char *src, const char *dst);
+static void fsync_file_and_parent_dir(const char *path);
+static bool file_contents_match(const char *file1, const char *file2);
+static void get_file_checksum(const char *file, int *checksumlen, uint8 *checksumbuf);
+
+/*
+ * _PG_archive_module_init
+ *
+ * Defines the module's GUC and returns its callbacks.
+ */
+void
+_PG_archive_module_init(ArchiveModuleCallbacks *cb)
+{
+	if (!process_archive_library_in_progress)
+		ereport(ERROR,
+				(errmsg("\"basic_archive\" can only be loaded via \"archive_library\"")));
+
+	DefineCustomStringVariable("basic_archive.archive_directory",
+							   gettext_noop("Archive file destination directory."),
+							   NULL,
+							   &archive_directory,
+							   "",
+							   PGC_SIGHUP,
+							   GUC_NOT_IN_SAMPLE,
+							   check_archive_directory, NULL, NULL);
+
+	EmitWarningsOnPlaceholders("basic_archive");
+
+	cb->check_configured_cb = basic_archive_configured;
+	cb->archive_file_cb = basic_archive_file;
+}
+
+/*
+ * check_archive_directory
+ *
+ * Checks that the provided archive directory exists.
+ */
+static bool
+check_archive_directory(char **newval, void **extra, GucSource source)
+{
+	struct stat st;
+
+	/*
+	 * The default value is an empty string, so we have to accept that value.
+	 * Our check_configured callback also checks for this and prevents archiving
+	 * from proceeding if it is still empty.
+	 */
+	if (*newval == NULL || *newval[0] == '\0')
+		return true;
+
+	/*
+	 * Do a basic sanity check that the specified archive directory exists.  It
+	 * could be removed at some point in the future, so we still need to be
+	 * prepared for it not to exist in the actual archiving logic.
+	 */
+	if (stat(*newval, &st) != 0 || !S_ISDIR(st.st_mode))
+	{
+		GUC_check_errdetail("specified archive directory does not exist");
+		return false;
+	}
+
+	return true;
+}
+
+/*
+ * basic_archive_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_archive_configured(void)
+{
+	return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_archive_file
+ *
+ * Archives one file.
+ */
+static bool
+basic_archive_file(const char *file, const char *path)
+{
+	char destination[MAXPGPATH];
+	char temp[MAXPGPATH];
+	struct stat st;
+
+	Assert(file != NULL);
+	Assert(path != NULL);
+
+	ereport(DEBUG3,
+			(errmsg("archiving \"%s\" via basic_archive", file)));
+
+#define TEMP_FILE_NAME ("archtemp")
+
+	Assert(MAX_XFN_CHARS >= strlen(TEMP_FILE_NAME));
+	if (strlen(archive_directory) + MAX_XFN_CHARS + 2 >= MAXPGPATH)
+	{
+		ereport(WARNING,
+				(errmsg("archive destination path too long")));
+		return false;
+	}
+
+	snprintf(destination, MAXPGPATH, "%s/%s", archive_directory, file);
+	snprintf(temp, MAXPGPATH, "%s/%s", archive_directory, TEMP_FILE_NAME);
+
+	/*
+	 * First, check if the file has already been archived.
+	 *
+	 * If the archive file already exists, check if its content matches the to-
+	 * be-archived file.  If the files match, we assume that the archiver
+	 * previously crashed at an unfortunate time and that we can safely proceed.
+	 * If the files do not match, something might be wrong, so we just fail.
+	 */
+	if (stat(destination, &st) == 0)
+	{
+		if (file_contents_match(path, destination))
+		{
+			ereport(DEBUG3,
+					(errmsg("archive file \"%s\" already exists with matching contents",
+							destination)));
+			fsync_file_and_parent_dir(destination);
+			return true;
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("archive file \"%s\" already exists with different contents",
+							destination)));
+			return false;
+		}
+	}
+	else if (errno != ENOENT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat file \"%s\": %m", destination)));
+
+	/*
+	 * Remove pre-existing temporary file, if one exists.
+	 */
+	if (unlink(temp) != 0 && errno != ENOENT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not unlink file \"%s\": %m", temp)));
+
+	/*
+	 * Copy the file to its temporary destination.
+	 */
+	copy_file(path, temp);
+
+	/*
+	 * Sync the temporary file to disk and move it to its final destination.
+	 */
+	(void) durable_rename(temp, destination, ERROR);
+
+	ereport(DEBUG1,
+			(errmsg("archived \"%s\" via basic_archive", file)));
+
+	return true;
+}
+
+/*
+ * copy_file
+ *
+ * Copies the contents of src to dst.
+ */
+static void
+copy_file(const char *src, const char *dst)
+{
+	int srcfd;
+	int dstfd;
+	char *buf;
+
+	Assert(src != NULL);
+	Assert(dst != NULL);
+
+#define COPY_BUF_SIZE (64 * 1024)
+
+	buf = palloc(COPY_BUF_SIZE);
+
+	srcfd = OpenTransientFile(src, O_RDONLY | PG_BINARY);
+	if (srcfd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open file \"%s\": %m", src)));
+
+	dstfd = OpenTransientFile(dst, O_RDWR | O_CREAT | O_EXCL | PG_BINARY);
+	if (dstfd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open file \"%s\": %m", dst)));
+
+	for (;;)
+	{
+		int nbytes = read(srcfd, buf, COPY_BUF_SIZE);
+		if (nbytes < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", src)));
+
+		if (nbytes == 0)
+			break;
+
+		errno = 0;
+		if ((int) write(dstfd, buf, nbytes) != nbytes)
+		{
+			/* if write didn't set errno, assume problem is no disk space */
+			if (errno == 0)
+				errno = ENOSPC;
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not write to file \"%s\": %m", dst)));
+		}
+	}
+
+	if (CloseTransientFile(dstfd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", dst)));
+
+	if (CloseTransientFile(srcfd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", src)));
+
+	pfree(buf);
+}
+
+/*
+ * fsync_file_and_parent_dir
+ *
+ * Flushes the given file and its parent directory to disk.
+ */
+static void
+fsync_file_and_parent_dir(const char *path)
+{
+	char parentpath[MAXPGPATH];
+
+	fsync_fname_ext(path, false, false, ERROR);
+
+	strlcpy(parentpath, path, MAXPGPATH);
+	get_parent_directory(parentpath);
+
+	/*
+	 * get_parent_directory() returns an empty string if the input argument is
+	 * just a file name (see comments in path.c), so handle that as being the
+	 * current directory.
+	 */
+	if (strlen(parentpath) == 0)
+		strlcpy(parentpath, ".", MAXPGPATH);
+
+	(void) fsync_fname_ext(parentpath, true, false, ERROR);
+}
+
+/*
+ * get_file_checksum
+ *
+ * Calculates a basic checksum for the given file.  The checksums returned by
+ * this function are not cryptographically strong.
+ */
+static void
+get_file_checksum(const char *file, int *checksumlen, uint8 *checksumbuf)
+{
+	int fd;
+	uint8 *buf;
+	pg_checksum_context checksum_ctx;
+
+	Assert(file != NULL);
+	Assert(checksumlen != NULL);
+	Assert(checksumbuf != NULL);
+
+#define CHECKSUM_BUF_SIZE (64 * 1024)
+
+	buf = palloc(CHECKSUM_BUF_SIZE);
+
+	fd = OpenTransientFile(file, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open file \"%s\": %m", file)));
+
+	if (pg_checksum_init(&checksum_ctx, CHECKSUM_TYPE_CRC32C) < 0)
+		ereport(ERROR,
+				(errmsg("could not initialized checksum of file \"%s\"", file)));
+
+	for (;;)
+	{
+		int nbytes = read(fd, buf, CHECKSUM_BUF_SIZE);
+		if (nbytes < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", file)));
+
+		if (nbytes == 0)
+			break;
+
+		if (pg_checksum_update(&checksum_ctx, buf, nbytes) < 0)
+			ereport(ERROR,
+					(errmsg("could not update checksum of file \"%s\"", file)));
+	}
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", file)));
+
+	*checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
+	if (*checksumlen < 0)
+		ereport(ERROR,
+				(errmsg("could not finalize checksum of file \"%s\"", file)));
+
+	pfree(buf);
+}
+
+/*
+ * file_contents_match
+ *
+ * This determines whether the two given files have the same contents by
+ * calculating and comparing their checksums.  If the files have the same
+ * contents, this function returns true.  If the files have different contents,
+ * this function will typically return false, but may in rare circumstances
+ * return true due to a checksum collision.  This risk of this function
+ * returning true for files with different contents is considered negligible.
+ */
+static bool
+file_contents_match(const char *file1, const char *file2)
+{
+	uint8 checksumbuf1[PG_CHECKSUM_MAX_LENGTH];
+	uint8 checksumbuf2[PG_CHECKSUM_MAX_LENGTH];
+	int checksumlen1 = 0;
+	int checksumlen2 = 0;
+
+	get_file_checksum(file1, &checksumlen1, checksumbuf1);
+	get_file_checksum(file2, &checksumlen2, checksumbuf2);
+
+	if (checksumlen1 != checksumlen2)
+		return false;
+
+	return memcmp(checksumbuf1, checksumbuf2, checksumlen1) == 0;
+}
-- 
2.16.6

