From c46c56e7b86e26a63f9b0b638d44558f2af93b8d Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 13 Nov 2025 12:59:56 +0900 Subject: [PATCH v5] Add test module test_large_files --- src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_large_files/Makefile | 19 ++ src/test/modules/test_large_files/README | 53 ++++ src/test/modules/test_large_files/meson.build | 29 ++ .../t/001_windows_large_files.pl | 65 +++++ .../test_large_files--1.0.sql | 36 +++ .../test_large_files/test_large_files.c | 270 ++++++++++++++++++ .../test_large_files/test_large_files.control | 5 + .../log/regress_log_001_windows_large_files | 1 + 10 files changed, 480 insertions(+) create mode 100644 src/test/modules/test_large_files/Makefile create mode 100644 src/test/modules/test_large_files/README create mode 100644 src/test/modules/test_large_files/meson.build create mode 100644 src/test/modules/test_large_files/t/001_windows_large_files.pl create mode 100644 src/test/modules/test_large_files/test_large_files--1.0.sql create mode 100644 src/test/modules/test_large_files/test_large_files.c create mode 100644 src/test/modules/test_large_files/test_large_files.control create mode 100644 src/test/modules/test_large_files/tmp_check/log/regress_log_001_windows_large_files diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 902a79541010..442713428791 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -29,6 +29,7 @@ SUBDIRS = \ test_int128 \ test_integerset \ test_json_parser \ + test_large_files \ test_lfind \ test_lwlock_tranches \ test_misc \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 14fc761c4cfa..95af220a4d97 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -28,6 +28,7 @@ subdir('test_ginpostinglist') subdir('test_int128') subdir('test_integerset') subdir('test_json_parser') +subdir('test_large_files') subdir('test_lfind') subdir('test_lwlock_tranches') subdir('test_misc') diff --git a/src/test/modules/test_large_files/Makefile b/src/test/modules/test_large_files/Makefile new file mode 100644 index 000000000000..f9fa977797d0 --- /dev/null +++ b/src/test/modules/test_large_files/Makefile @@ -0,0 +1,19 @@ +# src/test/modules/test_large_files/Makefile + +MODULES = test_large_files + +EXTENSION = test_large_files +DATA = test_large_files--1.0.sql + +TAP_TESTS = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_large_files +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_large_files/README b/src/test/modules/test_large_files/README new file mode 100644 index 000000000000..d7caae49e6a3 --- /dev/null +++ b/src/test/modules/test_large_files/README @@ -0,0 +1,53 @@ +Test Module for Windows Large File I/O + +This test module provides functions to test PostgreSQL's ability to +handle files larger than 4GB on Windows. + +Requirements + +- Windows platform +- PostgreSQL built with segment size greater than 2GB +- NTFS filesystem (for sparse file support) + +Functions + +test_create_sparse_file(filename text, size_gb int) RETURNS boolean + +Creates a sparse file of the specified size in gigabytes. This allows +testing large offsets without actually writing gigabytes of data to +disk. + +test_sparse_write_read(filename text, offset_gb float8, test_data text) +RETURNS boolean + +Writes test data at the specified offset (in GB) using PostgreSQL's VFD +layer (FileWrite), then reads it back using FileRead to verify basic I/O +functionality. + +test_verify_offset_native(filename text, offset_gb float8, expected_data +text) RETURNS boolean + +Critical for validation: Uses native Windows APIs (ReadFile with proper +OVERLAPPED structure) to verify that data written by PostgreSQL is +actually at the correct offset. This catches bugs where both write and +read might use the same incorrect offset calculation (making a broken +test appear to pass). + +Without this verification, a test could pass even with broken offset +handling if both FileWrite and FileRead make the same mistake. + +What the Test Verifies + +1. Sparse file creation works on Windows +2. PostgreSQL's FileWrite can write at offsets > 4GB +3. PostgreSQL's FileRead can read from offsets > 4GB +4. Data is actually at the correct offset (verified with native Windows + APIs) + +The native verification step is critical because without it, a test +could pass even with broken offset handling. For example, if both +FileWrite and FileRead truncate offsets to 32 bits, writing at 4.5GB +would actually write at ~512MB, and reading at 4.5GB would read from +~512MB - the test would find matching data but at the wrong location. +The native verification catches this by independently checking the +actual file offset. diff --git a/src/test/modules/test_large_files/meson.build b/src/test/modules/test_large_files/meson.build new file mode 100644 index 000000000000..c755e2cf16d0 --- /dev/null +++ b/src/test/modules/test_large_files/meson.build @@ -0,0 +1,29 @@ +# src/test/modules/test_large_files/meson.build + +test_large_files_sources = files( + 'test_large_files.c', +) + +if host_system == 'windows' + test_large_files = shared_module('test_large_files', + test_large_files_sources, + kwargs: pg_test_mod_args, + ) + test_install_libs += test_large_files + + test_install_data += files( + 'test_large_files.control', + 'test_large_files--1.0.sql', + ) + + tests += { + 'name': 'test_large_files', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'tap': { + 'tests': [ + 't/001_windows_large_files.pl', + ], + }, + } +endif diff --git a/src/test/modules/test_large_files/t/001_windows_large_files.pl b/src/test/modules/test_large_files/t/001_windows_large_files.pl new file mode 100644 index 000000000000..2fb0ef5e36bf --- /dev/null +++ b/src/test/modules/test_large_files/t/001_windows_large_files.pl @@ -0,0 +1,65 @@ +#!/usr/bin/perl +# Copyright (c) 2025, PostgreSQL Global Development Group + +=pod + +=head1 NAME + +001_windows_large_files.pl - Test Windows support for files >4GB + +=head1 SYNOPSIS + + prove src/test/modules/test_large_files/t/001_windows_large_files.pl + +=head1 DESCRIPTION + +This test verifies that PostgreSQL on Windows can correctly handle file +operations at offsets beyond 4GB. This requires PostgreSQL to be +built with a segment size greater than 2GB. + +The test uses sparse files to avoid actually writing gigabytes of data. + +=cut + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use File::Spec; +use File::Temp; + +if ($^O ne 'MSWin32') +{ + plan skip_all => 'test is Windows-specific'; +} + +plan tests => 4; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION test_large_files;'); +pass("test_large_files extension loaded"); + +my $tempdir = File::Temp->newdir(); +my $testfile = File::Spec->catfile($tempdir, 'large_file_test.dat'); + +note "Test file: $testfile"; + +my $create_result = $node->safe_psql('postgres', + "SELECT test_create_sparse_file('$testfile', 5);"); +is($create_result, 't', "Created 5GB sparse file"); + +my $test_4_5gb = $node->safe_psql('postgres', + "SELECT test_sparse_write_read('$testfile', 4.5, 'TEST_DATA_AT_4.5GB');"); +is($test_4_5gb, 't', "Write/read successful at 4.5GB offset"); + +my $verify_4_5gb = $node->safe_psql('postgres', + "SELECT test_verify_offset_native('$testfile', 4.5, 'TEST_DATA_AT_4.5GB');"); +is($verify_4_5gb, 't', "Native verification confirms data at correct 4.5GB offset"); + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_large_files/test_large_files--1.0.sql b/src/test/modules/test_large_files/test_large_files--1.0.sql new file mode 100644 index 000000000000..c4db84106c8d --- /dev/null +++ b/src/test/modules/test_large_files/test_large_files--1.0.sql @@ -0,0 +1,36 @@ +-- src/test/modules/test_large_files/test_large_files--1.0.sql + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_large_files" to load this file. \quit + +-- +-- test_create_sparse_file(filename text, size_gb int) returns boolean +-- +-- Creates a sparse file for testing. Windows only. +-- +CREATE FUNCTION test_create_sparse_file(filename text, size_gb int) +RETURNS boolean +AS 'MODULE_PATHNAME', 'test_create_sparse_file' +LANGUAGE C STRICT; + +-- +-- test_sparse_write_read(filename text, offset_gb numeric, test_data text) returns boolean +-- +-- Writes data at a large offset and reads it back to verify correctness. +-- Tests pg_pwrite/pg_pread with offsets beyond 2GB and 4GB. Windows only. +-- +CREATE FUNCTION test_sparse_write_read(filename text, offset_gb float8, test_data text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'test_sparse_write_read' +LANGUAGE C STRICT; + +-- +-- test_verify_offset_native(filename text, offset_gb numeric, expected_data text) returns boolean +-- +-- Uses native Windows APIs to verify data is at the correct offset. +-- This ensures PostgreSQL's I/O didn't write to a wrapped/incorrect offset. +-- +CREATE FUNCTION test_verify_offset_native(filename text, offset_gb float8, expected_data text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'test_verify_offset_native' +LANGUAGE C STRICT; diff --git a/src/test/modules/test_large_files/test_large_files.c b/src/test/modules/test_large_files/test_large_files.c new file mode 100644 index 000000000000..623d2d214cde --- /dev/null +++ b/src/test/modules/test_large_files/test_large_files.c @@ -0,0 +1,270 @@ +/* src/test/modules/test_large_files/test_large_files.c */ + +#include "postgres.h" + +#include "fmgr.h" +#include "storage/fd.h" +#include "utils/builtins.h" + +#ifdef WIN32 +#include +#include +#endif + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(test_sparse_write_read); +PG_FUNCTION_INFO_V1(test_create_sparse_file); +PG_FUNCTION_INFO_V1(test_verify_offset_native); + +/* + * test_verify_offset_native(filename text, offset_gb numeric, expected_data text) returns boolean + * + * Uses native Windows APIs to read data at the specified offset and verify it matches. + * This ensures PostgreSQL's I/O functions wrote to the CORRECT offset, not a wrapped one. + * Windows only. + */ +Datum +test_verify_offset_native(PG_FUNCTION_ARGS) +{ +#ifdef WIN32 + text *filename_text = PG_GETARG_TEXT_PP(0); + float8 offset_gb = PG_GETARG_FLOAT8(1); + text *expected_text = PG_GETARG_TEXT_PP(2); + char *filename; + char *expected_data; + char *read_buffer; + int expected_len; + int64 offset; + HANDLE hFile; + OVERLAPPED overlapped = {0}; + DWORD bytesRead; + bool success = false; + + filename = text_to_cstring(filename_text); + expected_data = text_to_cstring(expected_text); + expected_len = strlen(expected_data) + 1; + + /* Calculate offset in bytes */ + offset = (int64) (offset_gb * 1024.0 * 1024.0 * 1024.0); + + /* Open file with native Windows API */ + hFile = CreateFile(filename, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (hFile == INVALID_HANDLE_VALUE) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for verification: %lu", + filename, GetLastError()))); + + /* Set up OVERLAPPED structure with proper 64-bit offset */ + overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); + overlapped.OffsetHigh = (DWORD)(offset >> 32); + + /* Allocate read buffer */ + read_buffer = palloc(expected_len); + + /* Read using native Windows API */ + if (!ReadFile(hFile, read_buffer, expected_len, &bytesRead, &overlapped)) + { + DWORD error = GetLastError(); + CloseHandle(hFile); + pfree(read_buffer); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("native ReadFile failed at offset %lld: %lu", + offset, error))); + } + + if (bytesRead != expected_len) + { + CloseHandle(hFile); + pfree(read_buffer); + ereport(ERROR, + (errmsg("native ReadFile read %lu bytes, expected %d", + bytesRead, expected_len))); + } + + /* Verify data matches */ + success = (memcmp(expected_data, read_buffer, expected_len) == 0); + + pfree(read_buffer); + CloseHandle(hFile); + + if (!success) + ereport(ERROR, + (errmsg("data mismatch at offset %lld: PostgreSQL wrote to wrong location", + offset))); + + PG_RETURN_BOOL(success); +#else + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("this test is only supported on Windows"))); + PG_RETURN_BOOL(false); +#endif +} + +/* + * test_create_sparse_file(filename text, size_gb int) returns boolean + * + * Creates a sparse file of the specified size in gigabytes. + * Windows only. + */ +Datum +test_create_sparse_file(PG_FUNCTION_ARGS) +{ +#ifdef WIN32 + text *filename_text = PG_GETARG_TEXT_PP(0); + int32 size_gb = PG_GETARG_INT32(1); + char *filename; + HANDLE hFile; + DWORD bytesReturned; + LARGE_INTEGER fileSize; + bool success = false; + + filename = text_to_cstring(filename_text); + + /* Open/create the file */ + hFile = CreateFile(filename, + GENERIC_WRITE, + 0, + NULL, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (hFile == INVALID_HANDLE_VALUE) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %lu", + filename, GetLastError()))); + + /* Mark as sparse */ + if (!DeviceIoControl(hFile, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, + &bytesReturned, NULL)) + { + CloseHandle(hFile); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set file sparse: %lu", GetLastError()))); + } + + /* Set file size */ + fileSize.QuadPart = (int64) size_gb * 1024 * 1024 * 1024; + if (!SetFilePointerEx(hFile, fileSize, NULL, FILE_BEGIN)) + { + CloseHandle(hFile); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set file pointer: %lu", GetLastError()))); + } + + if (!SetEndOfFile(hFile)) + { + CloseHandle(hFile); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set end of file: %lu", GetLastError()))); + } + + success = true; + CloseHandle(hFile); + + PG_RETURN_BOOL(success); +#else + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("sparse file test only supported on Windows"))); + PG_RETURN_BOOL(false); +#endif +} + +/* + * test_sparse_write_read(filename text, offset_gb numeric, test_data text) returns boolean + * + * Writes test data at the specified offset (in GB) and reads it back to verify. + * Tests that pg_pwrite and pg_pread work correctly with large offsets. + * Windows only. + */ +Datum +test_sparse_write_read(PG_FUNCTION_ARGS) +{ +#ifdef WIN32 + text *filename_text = PG_GETARG_TEXT_PP(0); + float8 offset_gb = PG_GETARG_FLOAT8(1); + text *test_data_text = PG_GETARG_TEXT_PP(2); + char *filename; + char *test_data; + char *read_buffer; + int test_data_len; + pgoff_t offset; + int fd; + ssize_t written; + ssize_t nread; + bool success = false; + + filename = text_to_cstring(filename_text); + test_data = text_to_cstring(test_data_text); + test_data_len = strlen(test_data) + 1; /* include null terminator */ + + /* Calculate offset in bytes */ + offset = (pgoff_t) (offset_gb * 1024.0 * 1024.0 * 1024.0); + + /* Open the file using PostgreSQL's VFD layer */ + fd = BasicOpenFile(filename, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", filename))); + + /* Write test data at the specified offset using pg_pwrite */ + written = pg_pwrite(fd, test_data, test_data_len, offset); + if (written != test_data_len) + { + close(fd); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file at offset %lld: wrote %zd of %d bytes", + (long long) offset, written, test_data_len))); + } + + /* Allocate buffer for reading */ + read_buffer = palloc(test_data_len); + + /* Read back the data using pg_pread */ + nread = pg_pread(fd, read_buffer, test_data_len, offset); + if (nread != test_data_len) + { + close(fd); + pfree(read_buffer); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from file at offset %lld: read %zd of %d bytes", + (long long) offset, nread, test_data_len))); + } + + /* Verify data matches */ + success = (memcmp(test_data, read_buffer, test_data_len) == 0); + + pfree(read_buffer); + close(fd); + + if (!success) + ereport(ERROR, + (errmsg("data mismatch: read data does not match written data"))); + + PG_RETURN_BOOL(success); +#else + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("this test is only supported on Windows"))); + PG_RETURN_BOOL(false); +#endif +} diff --git a/src/test/modules/test_large_files/test_large_files.control b/src/test/modules/test_large_files/test_large_files.control new file mode 100644 index 000000000000..9b0a30974b95 --- /dev/null +++ b/src/test/modules/test_large_files/test_large_files.control @@ -0,0 +1,5 @@ +# test_large_files extension +comment = 'Test module for large file I/O on Windows' +default_version = '1.0' +module_pathname = '$libdir/test_large_files' +relocatable = true diff --git a/src/test/modules/test_large_files/tmp_check/log/regress_log_001_windows_large_files b/src/test/modules/test_large_files/tmp_check/log/regress_log_001_windows_large_files new file mode 100644 index 000000000000..6d1526ee93d9 --- /dev/null +++ b/src/test/modules/test_large_files/tmp_check/log/regress_log_001_windows_large_files @@ -0,0 +1 @@ +[12:57:48.543](0.006s) 1..0 # SKIP test is Windows-specific -- 2.51.0