From fb877cdd7ed13cb41376049fadcf19f84a27094f Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Mon, 18 May 2026 23:05:36 +0300 Subject: [PATCH v1] Add Github Actions as CI provider --- .github/workflows/main.yml | 747 +++++++++++++++++++ src/tools/ci/ci_provider_helpers.sh | 443 +++++++++++ src/tools/ci/ci_provider_windows_helpers.ps1 | 93 +++ 3 files changed, 1283 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100755 src/tools/ci/ci_provider_helpers.sh create mode 100644 src/tools/ci/ci_provider_windows_helpers.ps1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000..6366ac3d8e3 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,747 @@ +# GitHub Actions CI workflow + +name: Postgres Github Actions CI + +on: + push: + branches: [ "*" ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # Never cancel in-progress runs on master to ensure all commits are tested. + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +env: + PGCTLTIMEOUT: 120 + PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance oauth + CCACHE_MAXSIZE: 250M + CI_OS_ONLY_JOBS: sanitycheck linux macos windows mingw compilerwarnings + + MBUILD_TARGET: all testprep + MTEST_ARGS: --print-errorlogs --no-rebuild -C build + CHECK: check-world PROVE_FLAGS=--timer + CHECKFLAGS: -Otarget + + MESON_COMMON_PG_CONFIG_ARGS: -Dcassert=true -Dinjection_points=true + MESON_COMMON_FEATURES: >- + -Dauto_features=disabled + -Dldap=enabled + -Dssl=openssl + -Dtap_tests=enabled + -Dplperl=enabled + -Dplpython=enabled + -Ddocs=enabled + -Dicu=enabled + -Dlibxml=enabled + -Dlibxslt=enabled + -Dlz4=enabled + -Dpltcl=enabled + -Dreadline=enabled + -Dzlib=enabled + -Dzstd=enabled + + LINUX_CONFIGURE_FEATURES: >- + --with-gssapi + --with-icu + --with-ldap + --with-libcurl + --with-libxml + --with-libxslt + --with-llvm + --with-lz4 + --with-pam + --with-perl + --with-python + --with-selinux + --with-ssl=openssl + --with-systemd + --with-tcl --with-tclconfig=/usr/lib/tcl8.6/ + --with-uuid=ossp + --with-zstd + + ARTIFACT_RETENTION_DAYS: 30 + MESON_LOG_PATHS: | + build*/testrun/**/*.log + build*/testrun/**/*.diffs + build*/testrun/**/regress_log_* + build*/meson-logs/*.txt + + +jobs: + # Add 'ci-os-only: ...' to the commit description to run + # only specific jobs. Valid names: + # sanitycheck, linux, macos, windows, mingw, compilerwarnings + # Jobs can be space or comma-separated. + # Example: 'ci-os-only: macos, linux' runs only macOS and Linux jobs. + # If omitted, all jobs run. + ci-config: + name: CI Configuration + runs-on: ubuntu-24.04 + timeout-minutes: 1 + outputs: + run-sanitycheck: ${{ steps.parse.outputs.run-sanitycheck }} + run-linux: ${{ steps.parse.outputs.run-linux }} + run-macos: ${{ steps.parse.outputs.run-macos }} + run-windows: ${{ steps.parse.outputs.run-windows }} + run-mingw: ${{ steps.parse.outputs.run-mingw }} + run-compilerwarnings: ${{ steps.parse.outputs.run-compilerwarnings }} + steps: + - name: Parse ci-os-only from commit message + id: parse + env: + # NOTE: github.event.head_commit.message is null on pull_request + # events. This workflow only triggers on push, so this is fine. + # If a pull_request trigger is added later, the filter will be + # silently disabled unless this is reworked. + COMMIT_MSG: ${{ github.event.head_commit.message }} + run: | + set -e + CI_ONLY=$(echo "${COMMIT_MSG}" | grep -i '^ci-os-only:' | head -1 | sed 's/^[^:]*://' | tr ',' ' ' | xargs) + if [ -z "${CI_ONLY}" ]; then + echo "No ci-os-only filter found, all jobs enabled" + for name in ${CI_OS_ONLY_JOBS}; do + echo "run-${name}=true" >> "${GITHUB_OUTPUT}" + done + else + echo "ci-os-only filter: ${CI_ONLY}" + for name in ${CI_OS_ONLY_JOBS}; do + if echo " ${CI_ONLY} " | grep -qi " ${name} "; then + echo "run-${name}=true" >> "${GITHUB_OUTPUT}" + else + echo "run-${name}=false" >> "${GITHUB_OUTPUT}" + fi + done + fi + + + sanity-check: + name: SanityCheck + needs: ci-config + if: needs.ci-config.outputs.run-sanitycheck == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + + env: + BUILD_JOBS: 8 + TEST_JOBS: 8 + CCACHE_DIR: ${{ github.workspace }}/ccache_dir + CCACHE_MAXSIZE: 150M + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: System info + run: src/tools/ci/ci_provider_helpers.sh github_actions sysinfo + + - name: Install dependencies + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions install_packages linux minimal + + - name: Restore ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ github.job }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ github.job }}-${{ github.ref }}- + ccache-${{ github.job }}- + + - name: Setup cores + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions setup_cores linux /tmp/cores $(whoami) + + - name: Configure + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions configure_meson build \ + --buildtype=debug \ + --auto-features=disabled \ + -Ddefault_library=shared \ + -Dtap_tests=enabled + + - name: Build + run: src/tools/ci/ci_provider_helpers.sh github_actions build_meson build ${BUILD_JOBS} ${MBUILD_TARGET} + + - name: Test (minimal) + run: | + set -e + # Allow locked memory for AIO and core dumps for debugging failures. + sudo prlimit --pid $$ --memlock=unlimited:unlimited --core=unlimited:unlimited + src/tools/ci/ci_provider_helpers.sh github_actions sanity_test ${TEST_JOBS} ${MTEST_ARGS} + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: ${{ github.job }}-logs-${{ github.run_id }} + path: ${{ env.MESON_LOG_PATHS }} + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + + + linux-meson: + name: Linux - Ubuntu 24.04 - Meson + needs: [ci-config, sanity-check] + if: | + !cancelled() && + needs.ci-config.outputs.run-linux == 'true' && + needs.sanity-check.result != 'failure' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + + env: + BUILD_JOBS: 4 + TEST_JOBS: 8 + CCACHE_DIR: /tmp/ccache_dir + CCACHE_MAXSIZE: 400M + # NOTE: GitHub Actions does not allow referencing one env var from + # another in the same env block (neither workflow- nor job-level env + # is exposed via the ${{ env.* }} context here). The SANITIZER_FLAGS + # value is therefore duplicated in CFLAGS/CXXFLAGS/LDFLAGS below; keep + # them in sync. + SANITIZER_FLAGS: -fsanitize=alignment,undefined + CFLAGS: -Og -ggdb -fno-sanitize-recover=all -fsanitize=alignment,undefined + CXXFLAGS: -Og -ggdb -fno-sanitize-recover=all -fsanitize=alignment,undefined + LDFLAGS: -fsanitize=alignment,undefined + CC: ccache gcc + CXX: ccache g++ + UBSAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:verbosity=2 + ASAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:detect_leaks=0:detect_stack_use_after_return=0 + DEBUGINFOD_URLS: "https://debuginfod.debian.net" + LINUX_MESON_FEATURES: -Duuid=e2fs + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: System info + run: src/tools/ci/ci_provider_helpers.sh github_actions sysinfo + + - name: Install dependencies + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions install_packages linux full_32bit + + - name: Restore ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ github.job }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ github.job }}-${{ github.ref }}- + ccache-${{ github.job }}- + + - name: Setup cores + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions setup_cores linux /tmp/cores $(whoami) + + - name: Configure (64-bit) + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions configure_meson build \ + ${MESON_COMMON_PG_CONFIG_ARGS} \ + --buildtype=debug \ + ${LINUX_MESON_FEATURES} -Dllvm=enabled + + - name: Build (64-bit) + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions build_meson build ${BUILD_JOBS} ${MBUILD_TARGET} + ninja -C build -t missingdeps + + - name: Test (64-bit) + env: + PG_TEST_INITDB_EXTRA_OPTS: -c io_method=io_uring + run: | + set -e + # Allow locked memory for AIO and core dumps for debugging failures. + sudo prlimit --pid $$ --memlock=unlimited:unlimited --core=unlimited:unlimited + src/tools/ci/ci_provider_helpers.sh github_actions test_meson ${TEST_JOBS} ${MTEST_ARGS} + # so that we don't upload 64bit logs if 32bit fails + rm -rf build/ + + - name: Install conflicting 32-bit packages + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions install_packages linux full_32bit_extra + + - name: Configure (32-bit) + env: + CC: ccache gcc -m32 + CXX: ccache g++ -m32 + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions configure_meson build-32 \ + ${MESON_COMMON_PG_CONFIG_ARGS} \ + --buildtype=debug \ + --pkg-config-path /usr/lib/i386-linux-gnu/pkgconfig/ \ + -DPERL=perl5.38-i386-linux-gnu \ + ${LINUX_MESON_FEATURES} -Dlibnuma=disabled -Dliburing=disabled + + - name: Build (32-bit) + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions build_meson build-32 ${BUILD_JOBS} ${MBUILD_TARGET} + ninja -C build-32 -t missingdeps + + - name: Test (32-bit) + env: + # Override MTEST_ARGS to point at the 32-bit build directory. + MTEST_ARGS: --print-errorlogs --no-rebuild -C build-32 + run: | + set -e + # Allow locked memory for AIO and core dumps for debugging failures. + sudo prlimit --pid $$ --memlock=unlimited:unlimited --core=unlimited:unlimited + PYTHONCOERCECLOCALE=0 LANG=C \ + src/tools/ci/ci_provider_helpers.sh github_actions test_meson ${TEST_JOBS} ${MTEST_ARGS} + + - name: Core dump backtraces + if: failure() + run: src/tools/ci/cores_backtrace.sh linux /tmp/cores + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: ${{ github.job }}-logs-${{ github.run_id }} + path: ${{ env.MESON_LOG_PATHS }} + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + + + linux-autoconf: + name: Linux - Ubuntu 24.04 - Autoconf + needs: [ci-config, sanity-check] + if: | + !cancelled() && + needs.ci-config.outputs.run-linux == 'true' && + needs.sanity-check.result != 'failure' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + env: + BUILD_JOBS: 4 + TEST_JOBS: 8 + CCACHE_DIR: /tmp/ccache_dir + # NOTE: GitHub Actions does not allow referencing one env var from + # another in the same env block (neither workflow- nor job-level env + # is exposed via the ${{ env.* }} context here). The SANITIZER_FLAGS + # value is therefore duplicated in CFLAGS/CXXFLAGS/LDFLAGS below; keep + # them in sync. + SANITIZER_FLAGS: -fsanitize=address + CFLAGS: -Og -ggdb -fno-sanitize-recover=all -fsanitize=address + CXXFLAGS: -Og -ggdb -fno-sanitize-recover=all -fsanitize=address + LDFLAGS: -fsanitize=address + CC: ccache gcc + CXX: ccache g++ + UBSAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:verbosity=2 + ASAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:detect_leaks=0:detect_stack_use_after_return=0 + PG_TEST_PG_COMBINEBACKUP_MODE: --copy-file-range + DEBUGINFOD_URLS: "https://debuginfod.debian.net" + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: System info + run: src/tools/ci/ci_provider_helpers.sh github_actions sysinfo + + - name: Install dependencies + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions install_packages linux full + + - name: Restore ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ github.job }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ github.job }}-${{ github.ref }}- + ccache-${{ github.job }}- + + - name: Setup cores + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions setup_cores linux /tmp/cores $(whoami) + + - name: Setup hosts + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions setup_hosts + + - name: Configure + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions configure_autoconf \ + "${LINUX_CONFIGURE_FEATURES}" \ + --enable-cassert --enable-injection-points --enable-debug \ + --enable-tap-tests --enable-nls \ + --with-segsize-blocks=6 \ + --with-libnuma \ + --with-liburing \ + CLANG="ccache clang" + + - name: Build + run: src/tools/ci/ci_provider_helpers.sh github_actions build_autoconf ${BUILD_JOBS} world-bin + + - name: Test + run: | + set -e + # Allow locked memory for AIO and core dumps for debugging failures. + sudo prlimit --pid $$ --memlock=unlimited:unlimited --core=unlimited:unlimited + src/tools/ci/ci_provider_helpers.sh github_actions test_autoconf ${TEST_JOBS} ${CHECK} ${CHECKFLAGS} + + - name: Core dump backtraces + if: failure() + run: src/tools/ci/cores_backtrace.sh linux /tmp/cores + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: ${{ github.job }}-logs-${{ github.run_id }} + path: | + **/*.log + **/*.diffs + **/regress_log_* + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + + + macos-meson: + name: macOS - Sequoia - Meson + needs: [ci-config, sanity-check] + if: | + !cancelled() && + needs.ci-config.outputs.run-macos == 'true' && + needs.sanity-check.result != 'failure' + runs-on: macos-15 + timeout-minutes: 30 + + env: + BUILD_JOBS: 4 + TEST_JOBS: 8 + CCACHE_DIR: ${{ github.workspace }}/ccache + CFLAGS: -Og -ggdb + CXXFLAGS: -Og -ggdb + CC: ccache cc + CXX: ccache c++ + PG_TEST_PG_UPGRADE_MODE: --clone + PG_TEST_PG_COMBINEBACKUP_MODE: --clone + MESON_FEATURES: >- + -Dbonjour=enabled + -Ddtrace=enabled + -Dgssapi=enabled + -Dlibcurl=enabled + -Dnls=enabled + -Duuid=e2fs + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: System info + run: src/tools/ci/ci_provider_helpers.sh github_actions sysinfo + + - name: Install dependencies + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + sudo -E src/tools/ci/ci_provider_helpers.sh github_actions install_packages macos + echo "/opt/local/sbin:/opt/local/bin" >> ${GITHUB_PATH} + + - name: Restore ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ github.job }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ github.job }}-${{ github.ref }}- + ccache-${{ github.job }}- + + - name: Setup cores + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions setup_cores macos "${HOME}/cores" + + - name: Configure + env: + PKG_CONFIG_PATH: /opt/local/lib/pkgconfig + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions configure_meson build \ + ${MESON_COMMON_PG_CONFIG_ARGS} \ + --buildtype=debug \ + ${MESON_COMMON_FEATURES} ${MESON_FEATURES} \ + -Dextra_include_dirs=/opt/local/include \ + -Dextra_lib_dirs=/opt/local/lib + + - name: Build + run: src/tools/ci/ci_provider_helpers.sh github_actions build_meson build ${BUILD_JOBS} ${MBUILD_TARGET} + + - name: Test + run: | + set -e + ulimit -c unlimited + ulimit -n 1024 + src/tools/ci/ci_provider_helpers.sh github_actions test_meson ${TEST_JOBS} ${MTEST_ARGS} + + - name: Core dump backtraces + if: failure() + run: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores" + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: ${{ github.job }}-logs-${{ github.run_id }} + path: ${{ env.MESON_LOG_PATHS }} + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + + + windows-msvc: + name: Windows - Server 2022, VS 2022 - Meson & ninja + needs: [ci-config, sanity-check] + if: | + !cancelled() && + needs.ci-config.outputs.run-windows == 'true' && + needs.sanity-check.result != 'failure' + runs-on: windows-2022 + timeout-minutes: 60 + + env: + BUILD_JOBS: 4 + TEST_JOBS: 8 + PG_TEST_USE_UNIX_SOCKETS: 1 + PG_REGRESS_SOCK_DIR: ${{ github.workspace }} + MESON_FEATURES: >- + -Dcpp_args=/std:c++20 + -Dauto_features=disabled + -Dtap_tests=enabled + -Dldap=enabled + -Dssl=openssl + -Dplperl=enabled + -Dplpython=enabled + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: System info + run: src/tools/ci/ci_provider_windows_helpers.ps1 github_actions sysinfo + + - name: Disable Windows Defender + shell: powershell + run: src/tools/ci/ci_provider_windows_helpers.ps1 github_actions disable_defender + + - name: Install dependencies + shell: powershell + run: src/tools/ci/ci_provider_windows_helpers.ps1 github_actions install_packages + + - name: Setup hosts file + shell: powershell + run: src/tools/ci/ci_provider_windows_helpers.ps1 github_actions setup_hosts + + - name: Setup MSVC environment and configure + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + meson setup %MESON_COMMON_PG_CONFIG_ARGS% --backend ninja --buildtype debug -Db_pch=true -DTAR=c:\windows\system32\tar.exe %MESON_FEATURES% build + + - name: Build + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + ninja -C build -j%BUILD_JOBS% %MBUILD_TARGET% + ninja -C build -t missingdeps + + - name: Test + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + REM Grant Everyone full control on the build directory so that + REM postgres.exe can access data files after it drops admin + REM privileges via CreateRestrictedToken(). + icacls build /grant Everyone:(OI)(CI)(F) /T /Q + meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: ${{ github.job }}-logs-${{ github.run_id }} + path: | + ${{ env.MESON_LOG_PATHS }} + crashlog-*.txt + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + + + windows-mingw: + name: Windows - Server 2022, MinGW64 - Meson + needs: [ci-config, sanity-check] + if: | + !cancelled() && + needs.ci-config.outputs.run-mingw == 'true' && + needs.sanity-check.result != 'failure' + runs-on: windows-2022 + timeout-minutes: 60 + + defaults: + run: + shell: msys2 {0} + + env: + BUILD_JOBS: 4 + TEST_JOBS: 4 + CCACHE_MAXSIZE: 500M + CCACHE_SLOPPINESS: pch_defines,time_macros + CCACHE_DEPEND: 1 + PG_TEST_USE_UNIX_SOCKETS: 1 + PG_REGRESS_SOCK_DIR: ${{ github.workspace }} + + steps: + - name: Disable Windows Defender + shell: powershell + # Disable Windows Defender before installing packages to speed up the + # process. We cannot use the helper script here because it requires the + # repository to be checked out first, and checkout in turn requires + # MSYS2 to already be set up with all packages installed. To break this + # circular dependency, the Defender disabling logic is inlined below. + run: | + Set-MpPreference -DisableRealtimeMonitoring $true -SubmitSamplesConsent NeverSend -MAPSReporting Disable + # Verify Defender status + $status = Get-MpComputerStatus -ErrorAction SilentlyContinue + if ($status) { + Write-Host "RealTimeProtectionEnabled: $($status.RealTimeProtectionEnabled)" + Write-Host "AntivirusEnabled: $($status.AntivirusEnabled)" + } + + - name: Setup MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: true + install: >- + git bison flex make diffutils + mingw-w64-ucrt-x86_64-ccache + mingw-w64-ucrt-x86_64-docbook-xml + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-icu + mingw-w64-ucrt-x86_64-libbacktrace + mingw-w64-ucrt-x86_64-libxml2 + mingw-w64-ucrt-x86_64-libxslt + mingw-w64-ucrt-x86_64-lz4 + mingw-w64-ucrt-x86_64-make + mingw-w64-ucrt-x86_64-meson + mingw-w64-ucrt-x86_64-perl + mingw-w64-ucrt-x86_64-pkg-config + mingw-w64-ucrt-x86_64-python-cryptography + mingw-w64-ucrt-x86_64-python-pip + mingw-w64-ucrt-x86_64-python-pytest + mingw-w64-ucrt-x86_64-readline + mingw-w64-ucrt-x86_64-zlib + + - name: Checkout + uses: actions/checkout@v5 + + - name: System info + run: src/tools/ci/ci_provider_helpers.sh github_actions sysinfo + + - name: Install IPC::Run + run: src/tools/ci/ci_provider_helpers.sh github_actions install_ipc_run + + - name: Restore ccache + uses: actions/cache@v5 + with: + path: C:/msys64/ccache + key: ccache-${{ github.job }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ github.job }}-${{ github.ref }}- + ccache-${{ github.job }}- + + - name: Configure + env: + CCACHE_DIR: C:/msys64/ccache + # Keep -Dnls explicitly disabled (creates too many files, causes slowdown) + MESON_FEATURES: -Dnls=disabled + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions configure_meson build \ + ${MESON_COMMON_PG_CONFIG_ARGS} \ + -Ddebug=true -Doptimization=g -Db_pch=true \ + ${MESON_COMMON_FEATURES} ${MESON_FEATURES} \ + -DTAR=c:/windows/system32/tar.exe + + - name: Build + run: src/tools/ci/ci_provider_helpers.sh github_actions build_meson build ${BUILD_JOBS} ${MBUILD_TARGET} + + - name: Test + run: src/tools/ci/ci_provider_helpers.sh github_actions test_meson ${TEST_JOBS} ${MTEST_ARGS} + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: ${{ github.job }}-logs-${{ github.run_id }} + path: | + ${{ env.MESON_LOG_PATHS }} + crashlog-*.txt + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + + + compiler-warnings: + name: CompilerWarnings + needs: [ci-config, sanity-check] + if: | + !cancelled() && + needs.ci-config.outputs.run-compilerwarnings == 'true' && + needs.sanity-check.result != 'failure' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + + env: + BUILD_JOBS: 4 + CCACHE_DIR: /tmp/ccache_dir + CCACHE_MAXSIZE: 1G + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: System info + run: | + set -e + src/tools/ci/ci_provider_helpers.sh github_actions sysinfo + gcc -v + clang -v + + - name: Install dependencies + run: sudo src/tools/ci/ci_provider_helpers.sh github_actions install_packages linux compiler_warnings + + - name: Restore ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ github.job }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ github.job }}-${{ github.ref }}- + ccache-${{ github.job }}- + + - name: Setup Werror + run: src/tools/ci/ci_provider_helpers.sh github_actions compiler_warnings_setup + + - name: "Warnings: gcc (cassert off, dtrace on)" + run: src/tools/ci/ci_provider_helpers.sh github_actions compiler_warnings_gcc_no_cassert ${BUILD_JOBS} "${LINUX_CONFIGURE_FEATURES}" + + - name: "Warnings: gcc (cassert on, dtrace off)" + run: src/tools/ci/ci_provider_helpers.sh github_actions compiler_warnings_gcc_cassert ${BUILD_JOBS} "${LINUX_CONFIGURE_FEATURES}" + + - name: "Warnings: clang (cassert off, dtrace off)" + run: src/tools/ci/ci_provider_helpers.sh github_actions compiler_warnings_clang_no_cassert ${BUILD_JOBS} "${LINUX_CONFIGURE_FEATURES}" + + - name: "Warnings: clang (cassert on, dtrace on)" + run: src/tools/ci/ci_provider_helpers.sh github_actions compiler_warnings_clang_cassert ${BUILD_JOBS} "${LINUX_CONFIGURE_FEATURES}" + + - name: "Warnings: mingw cross-compile" + run: src/tools/ci/ci_provider_helpers.sh github_actions compiler_warnings_mingw_cross ${BUILD_JOBS} + + - name: "Warnings: docs build" + run: src/tools/ci/ci_provider_helpers.sh github_actions compiler_warnings_docs ${BUILD_JOBS} + + - name: "Warnings: headerscheck & cpluspluscheck" + run: src/tools/ci/ci_provider_helpers.sh github_actions compiler_warnings_headerscheck ${BUILD_JOBS} "${LINUX_CONFIGURE_FEATURES}" + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: ${{ github.job }}-logs-${{ github.run_id }} + path: | + **/*.log + **/*.diffs + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} diff --git a/src/tools/ci/ci_provider_helpers.sh b/src/tools/ci/ci_provider_helpers.sh new file mode 100755 index 00000000000..5d72cc1d84e --- /dev/null +++ b/src/tools/ci/ci_provider_helpers.sh @@ -0,0 +1,443 @@ +#!/bin/sh +# src/tools/ci/provider_helpers/main.sh - Unified CI entry point for PostgreSQL +# +# Usage: main.sh [args...] +# +# Providers: +# github_actions - GitHub Actions +# + +set -e + +PROVIDER="${1}" +TASK="${2}" +shift 2 + +############################################################################## +# Setup tasks +############################################################################## + +task_sysinfo() { + echo "=== System Information ===" + echo "Provider: ${PROVIDER}" + echo "User: $(whoami)" + echo "Hostname: $(hostname)" + echo "Kernel: $(uname -a)" + echo "" + echo "=== Resource Limits (Hard) ===" + ulimit -a -H + echo "" + echo "=== Resource Limits (Soft) ===" + ulimit -a -S + echo "" + echo "=== Environment ===" + export +} + +# install_packages [profile] +# os: linux | macos +# profile: minimal | full | full_32bit | compiler_warnings (linux only) +task_install_packages() { + local os="${1}" + local profile="${2:-full}" + + case "${os}" in + linux) _install_linux_packages "${profile}" ;; + macos) _install_macos_packages ;; + *) + echo "Error: unsupported OS '${os}' for install_packages" + exit 1 + ;; + esac +} + +_install_linux_packages() { + local profile="${1}" + + # Packages shared by every Linux build + local common="build-essential gdb git make meson ccache perl libperl-dev libipc-run-perl" + + # Full dependency set for autoconf / meson builds + local full=" + libreadline-dev libssl-dev zlib1g-dev libicu-dev + libxml2-dev libxslt1-dev gettext + liblz4-dev libzstd-dev + libldap-dev libkrb5-dev krb5-admin-server krb5-kdc krb5-user + libpam0g-dev uuid-dev libossp-uuid-dev + python3-dev libtcl8.6 tcl-dev + libcurl4-openssl-dev + llvm-dev clang + libselinux1-dev libsystemd-dev + systemtap-sdt-dev + libnuma-dev liburing-dev" + + apt-get update -qq + + case "${profile}" in + minimal) + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y -qq --no-install-recommends ${common} + ;; + full) + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y -qq --no-install-recommends ${common} ${full} + ;; + full_32bit) + dpkg --add-architecture i386 + apt-get update -qq + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y -qq --no-install-recommends --no-remove \ + ${common} ${full} \ + gcc-multilib \ + libicu-dev:i386 \ + libldap2-dev:i386 \ + liblz4-dev:i386 \ + libpam-dev:i386 \ + libperl-dev:i386 \ + libpython3-dev:i386 \ + libreadline-dev:i386 \ + libselinux-dev:i386 \ + libssl-dev:i386 \ + libsystemd-dev:i386 \ + libxml2-dev:i386 \ + libxslt1-dev:i386 \ + libzstd-dev:i386 \ + tcl-dev:i386 \ + uuid-dev:i386 + ;; + full_32bit_extra) + # Packages that conflict with their amd64 counterparts (they + # install arch-independent files like curl-config). Install these + # with --force-overwrite after the 64-bit build and tests are done, + # so the 64-bit toolchain is not affected. + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y -qq --no-install-recommends \ + -o Dpkg::Options::="--force-overwrite" \ + libcurl4-openssl-dev:i386 + ;; + compiler_warnings) + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y -qq --no-install-recommends \ + ${common} ${full} \ + gcc g++ clang \ + gcc-mingw-w64-x86-64-posix g++-mingw-w64-x86-64-posix \ + docbook-xml docbook-xsl xsltproc libxml2-utils + ;; + *) + echo "Error: unknown Linux package profile '${profile}'" + exit 1 + ;; + esac +} + +_install_macos_packages() { + local macports_version="2.10.1" + local macos_major_version + macos_major_version=$(sw_vers -productVersion | sed 's/\..*//') + + if [ ! -x /opt/local/bin/port ]; then + echo "=== Installing MacPorts ===" + # Fetch the .pkg URL from the GitHub releases API for this macOS version. + local macports_url + local api_response + # Need to use GITHUB_TOKEN, otherwise might get API rate limit + # exceeded for ${ip_adress}. Only send the Authorization header when + # GITHUB_TOKEN is non-empty: an empty token results in a malformed + # "Authorization: token " header that some GitHub endpoints reject. + if [ -n "${GITHUB_TOKEN:-}" ]; then + api_response=$(curl -sH "Authorization: token ${GITHUB_TOKEN}" \ + "https://api.github.com/repos/macports/macports-base/releases") + else + api_response=$(curl -s \ + "https://api.github.com/repos/macports/macports-base/releases") + fi + macports_url=$(echo "${api_response}" | python3 -c " +import json, sys, re +releases = json.loads(sys.stdin.read(), strict=False) +for rel in releases: + if not rel['tag_name'].startswith('v${macports_version}'): + continue + for asset in rel.get('assets', []): + if re.match(r'MacPorts-.*-${macos_major_version}-.*\.pkg\$', asset['name']): + print(asset['browser_download_url']) + sys.exit(0) +") + if [ -z "${macports_url}" ]; then + echo "Error: could not find MacPorts package for macOS ${macos_major_version}" + exit 1 + fi + echo "Downloading: ${macports_url}" + curl -fsSL -o /tmp/macports.pkg "${macports_url}" + installer -pkg /tmp/macports.pkg -target / + rm -f /tmp/macports.pkg + fi + + export PATH=/opt/local/sbin/:/opt/local/bin/:${PATH} + + echo "=== Installing packages via MacPorts ===" + port install -N \ + ccache icu kerberos5 lz4 meson openldap openssl \ + p5.34-io-tty p5.34-ipc-run python312 tcl zstd + /opt/local/bin/port select python3 python312 +} + + + +# setup_cores [core_dir] +task_setup_cores() { + local os="${1}" + local core_dir="${2:-/tmp/cores}" + local username="${3}" + + mkdir -p "${core_dir}" + chmod 770 "${core_dir}" + + case "${os}" in + linux) + local conf_file="/etc/security/limits.d/${username}.conf" + + sysctl kernel.core_pattern="${core_dir}/%e-%s-%p.core" + + touch "${conf_file}" && chown "${username}:${username}" "${conf_file}" + # Allow all users to create unlimited core dumps + echo '* - core unlimited' > "${conf_file}" + ;; + macos) + sysctl kern.corefile="${core_dir}/core.%P" + ;; + *) + echo "Error: unsupported OS '${os}' for setup_cores" + exit 1 + ;; + esac +} + +task_setup_hosts() { + echo "127.0.0.1 pg-loadbalancetest" >> /etc/hosts + echo "127.0.0.2 pg-loadbalancetest" >> /etc/hosts + echo "127.0.0.3 pg-loadbalancetest" >> /etc/hosts + echo "Updated /etc/hosts with load-balance test entries" +} + +# install_ipc_run +# Installs IPC::Run for Perl (used by MinGW CI). +task_install_ipc_run() { + echo "=== Installing IPC::Run ===" + echo "Check if IPC::Run is already installed, it shouldn't be at this point" + if perl -mIPC::Run -e 1 2>/dev/null; then echo "ERROR: IPC::Run already installed"; exit 1; fi + # MinGW CI tasks started failing after the package was updated from + # NJM/IPC-Run-20250809.0 to TODDR/IPC-Run-20260322.0. There is no way to + # install IPC::Run from an author without specifying the exact version number + # so install the latest working one. (NJM/IPC-Run-20250809.0.tar.gz). See: + # - https://postgr.es/m/CAN55FZ06xanSbJdHe-CurjX_qNuBWZDEvS1kAk36L38YCtZXnw%40mail.gmail.com + (echo; echo o conf recommends_policy 0; echo notest install NJM/IPC-Run-20250809.0.tar.gz) | cpan + # Check if IPC::Run is installed correctly + perl -mIPC::Run -e 1 +} + +############################################################################## +# Configure tasks +############################################################################## + +# configure_meson [meson_args...] +task_configure_meson() { + local build_dir="${1}" + shift + + meson setup "$(pwd)" "${build_dir}" "$@" +} + +# configure_autoconf [configure_args...] +task_configure_autoconf() { + local features="${1}" + shift + + # shellcheck disable=SC2086 + ./configure ${features} "$@" +} + +############################################################################## +# Build tasks +############################################################################## + +# build_meson +task_build_meson() { + local build_dir="${1}" + local jobs="${2}" + shift 2 + + ninja -C "${build_dir}" -j"${jobs}" "$@" +} + +# build_autoconf +task_build_autoconf() { + local jobs="${1}" + shift + + make -s -j"${jobs}" "$@" +} + +############################################################################## +# Test tasks +############################################################################## + +# test_meson +task_test_meson() { + local jobs="${1}" + shift + + ulimit -c unlimited + meson test "$@" --num-processes "${jobs}" +} + +# test_autoconf +task_test_autoconf() { + local jobs="${1}" + shift + + ulimit -c unlimited + make -s -j"${jobs}" "$@" +} + +# sanity_test +task_sanity_test() { + local jobs="${1}" + shift + + ulimit -c unlimited + meson test "$@" --suite setup + meson test "$@" --num-processes "${jobs}" \ + cube/regress pg_ctl/001_start_stop +} + +task_compiler_warnings_setup() { + echo "COPT=-Werror" > src/Makefile.custom +} + +# compiler_warnings_gcc_no_cassert +task_compiler_warnings_gcc_no_cassert() { + local jobs="${1}" + local features="${2}" + + echo "====== gcc, cassert off, dtrace on ======" + # shellcheck disable=SC2086 + ./configure --cache gcc-no-cassert.cache --enable-dtrace \ + ${features} CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" + make -s -j"${jobs}" clean + make -s -j"${jobs}" world-bin +} + +# compiler_warnings_gcc_cassert +task_compiler_warnings_gcc_cassert() { + local jobs="${1}" + local features="${2}" + + echo "====== gcc, cassert on, dtrace off ======" + # shellcheck disable=SC2086 + ./configure --cache gcc-cassert.cache --enable-cassert \ + ${features} CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" + make -s -j"${jobs}" clean + make -s -j"${jobs}" world-bin +} + +# compiler_warnings_clang_no_cassert +task_compiler_warnings_clang_no_cassert() { + local jobs="${1}" + local features="${2}" + + echo "====== clang, cassert off, dtrace off ======" + # shellcheck disable=SC2086 + ./configure --cache clang-no-cassert.cache \ + ${features} CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang" + make -s -j"${jobs}" clean + make -s -j"${jobs}" world-bin +} + +# compiler_warnings_clang_cassert +task_compiler_warnings_clang_cassert() { + local jobs="${1}" + local features="${2}" + + echo "====== clang, cassert on, dtrace on ======" + # shellcheck disable=SC2086 + ./configure --cache clang-cassert.cache --enable-cassert --enable-dtrace \ + ${features} CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang" + make -s -j"${jobs}" clean + make -s -j"${jobs}" world-bin +} + +# compiler_warnings_mingw_cross +task_compiler_warnings_mingw_cross() { + local jobs="${1}" + + ./configure --host=x86_64-w64-mingw32 --enable-cassert \ + --without-icu --without-zlib \ + CC="ccache x86_64-w64-mingw32-gcc" \ + CXX="ccache x86_64-w64-mingw32-g++" + make -s -j"${jobs}" clean + make -s -j"${jobs}" world-bin +} + +# compiler_warnings_docs +task_compiler_warnings_docs() { + local jobs="${1}" + + ./configure --cache docs.cache \ + CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" + make -s -j"${jobs}" clean + make -s -j"${jobs}" -C doc +} + +# compiler_warnings_headerscheck +task_compiler_warnings_headerscheck() { + local jobs="${1}" + local features="${2}" + + # shellcheck disable=SC2086 + ./configure ${features} --cache headerscheck.cache --quiet \ + CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" + make -s -j"${jobs}" clean + make -s -j"${jobs}" -k -Otarget headerscheck cpluspluscheck \ + EXTRAFLAGS='-fmax-errors=10' +} + +############################################################################## +# Dispatch +############################################################################## + +case "${TASK}" in + sysinfo) task_sysinfo "$@" ;; + install_packages) task_install_packages "$@" ;; + install_ipc_run) task_install_ipc_run "$@" ;; + setup_cores) task_setup_cores "$@" ;; + setup_hosts) task_setup_hosts "$@" ;; + configure_meson) task_configure_meson "$@" ;; + configure_autoconf) task_configure_autoconf "$@" ;; + build_meson) task_build_meson "$@" ;; + build_autoconf) task_build_autoconf "$@" ;; + test_meson) task_test_meson "$@" ;; + test_autoconf) task_test_autoconf "$@" ;; + sanity_test) task_sanity_test "$@" ;; + compiler_warnings_setup) task_compiler_warnings_setup "$@" ;; + compiler_warnings_gcc_no_cassert) task_compiler_warnings_gcc_no_cassert "$@" ;; + compiler_warnings_gcc_cassert) task_compiler_warnings_gcc_cassert "$@" ;; + compiler_warnings_clang_no_cassert) task_compiler_warnings_clang_no_cassert "$@" ;; + compiler_warnings_clang_cassert) task_compiler_warnings_clang_cassert "$@" ;; + compiler_warnings_mingw_cross) task_compiler_warnings_mingw_cross "$@" ;; + compiler_warnings_docs) task_compiler_warnings_docs "$@" ;; + compiler_warnings_headerscheck) task_compiler_warnings_headerscheck "$@" ;; + *) + echo "Error: unknown task '${TASK}'" + echo "" + echo "Available tasks:" + echo " sysinfo install_packages install_ipc_run setup_cores setup_hosts" + echo " configure_meson configure_autoconf" + echo " build_meson build_autoconf" + echo " test_meson test_autoconf sanity_test" + echo " compiler_warnings_setup compiler_warnings_gcc_no_cassert" + echo " compiler_warnings_gcc_cassert compiler_warnings_clang_no_cassert" + echo " compiler_warnings_clang_cassert compiler_warnings_mingw_cross" + echo " compiler_warnings_docs compiler_warnings_headerscheck" + exit 1 + ;; +esac diff --git a/src/tools/ci/ci_provider_windows_helpers.ps1 b/src/tools/ci/ci_provider_windows_helpers.ps1 new file mode 100644 index 00000000000..af9cdbc4431 --- /dev/null +++ b/src/tools/ci/ci_provider_windows_helpers.ps1 @@ -0,0 +1,93 @@ +# src/tools/ci/provider_helpers/main.ps1 +# +# Usage: main.ps1 [args...] +# +# Providers: +# github_actions - GitHub Actions +# + +param( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Provider, + + [Parameter(Mandatory = $true, Position = 1)] + [string]$Task, + + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$ExtraArgs +) + +$ErrorActionPreference = "Stop" + +function Invoke-Sysinfo { + Write-Host "=== System Information ===" + Write-Host "Provider: $Provider" + chcp + systeminfo + Get-PSDrive -PSProvider FileSystem + Get-ChildItem Env: | Sort-Object Name +} + +function Invoke-SetupHosts { + $hostsFile = "C:\Windows\System32\Drivers\etc\hosts" + Add-Content -Path $hostsFile -Value "127.0.0.1 pg-loadbalancetest" + Add-Content -Path $hostsFile -Value "127.0.0.2 pg-loadbalancetest" + Add-Content -Path $hostsFile -Value "127.0.0.3 pg-loadbalancetest" + Write-Host "Updated hosts file:" + Get-Content $hostsFile +} + +function Invoke-DisableDefender { + Set-MpPreference -DisableRealtimeMonitoring $true -SubmitSamplesConsent NeverSend -MAPSReporting Disable + # Verify Defender status + $status = Get-MpComputerStatus -ErrorAction SilentlyContinue + if ($status) { + Write-Host "RealTimeProtectionEnabled: $($status.RealTimeProtectionEnabled)" + Write-Host "AntivirusEnabled: $($status.AntivirusEnabled)" + } +} + +function Invoke-InstallPackages { + $before = $env:Path -split ';' + + # First install only strawberryperl with the --version option, + # otherwise --version tag applies to all packages. + choco install -y --no-progress --force ` + strawberryperl --version=5.42.0.1 + + choco install -y --no-progress ` + meson ` + ninja ` + winflexbison3 ` + diffutils ` + pkgconfiglite ` + openssl ` + xsltproc ` + zstandard ` + python3 + + # Refresh PATH using Chocolatey's helper to pick up Machine PATH changes + Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 + refreshenv + # Persist new PATH entries for subsequent steps. + # Deduplicate to avoid appending the same path multiple times when + # multiple installed packages contribute the same directory. + $after = $env:Path -split ';' + $new = $after | Where-Object { $_ -and $before -notcontains $_ } | Select-Object -Unique + Write-Host "=== New PATH entries ===" + $new | ForEach-Object { Write-Host " $_" } + $new | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append +} + +switch ($Task) { + "sysinfo" { Invoke-Sysinfo } + "setup_hosts" { Invoke-SetupHosts } + "disable_defender" { Invoke-DisableDefender } + "install_packages" { Invoke-InstallPackages } + default { + Write-Error "Unknown task: $Task" + Write-Host "" + Write-Host "Available tasks: sysinfo setup_hosts disable_defender install_packages" + exit 1 + } +} -- 2.43.0