From 32031474861f410130522fd6ae647cbdc8b352ff Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 20 Apr 2022 12:08:09 -0700 Subject: [PATCH v10 16/16] meson: ci: Build both with meson and as before Author: Andres Freund Author: nbyavuz Author: Justin Pryzby --- .cirrus.yml | 545 +++++++++++++++++- src/tools/ci/docker/linux_centos7 | 57 ++ src/tools/ci/docker/linux_centos8 | 54 ++ src/tools/ci/docker/linux_fedora_rawhide | 49 ++ src/tools/ci/docker/linux_opensuse_tumbleweed | 56 ++ 5 files changed, 738 insertions(+), 23 deletions(-) create mode 100644 src/tools/ci/docker/linux_centos7 create mode 100644 src/tools/ci/docker/linux_centos8 create mode 100644 src/tools/ci/docker/linux_fedora_rawhide create mode 100644 src/tools/ci/docker/linux_opensuse_tumbleweed diff --git a/.cirrus.yml b/.cirrus.yml index f23d6cae552..bf20f85f8ce 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -35,10 +35,12 @@ on_failure: &on_failure - "**/*.diffs" - "**/regress_log_*" type: text/plain + meson_test_artifacts: + path: "build/meson-logs/testlog.junit.xml" + type: text/xml + format: junit -task: - name: FreeBSD - 13 - +freebsd_template: &freebsd_template env: # FreeBSD on GCP is slow when running with larger number of CPUS / # jobs. Using one more job than cpus seems to work best. @@ -80,6 +82,13 @@ task: setup_additional_packages_script: | #pkg install -y ... + +task: + <<: *freebsd_template + + name: FreeBSD - 13 - Autoconf + trigger_type: manual + # NB: Intentionally build without --with-llvm. The freebsd image size is # already large enough to make VM startup slow, and even without llvm # freebsd already takes longer than other platforms except for windows. @@ -125,6 +134,30 @@ task: cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores +task: + <<: *freebsd_template + + name: FreeBSD - 13 - Meson + + configure_script: + - su postgres -c 'meson setup --buildtype debug -Dcassert=true -Dssl=openssl -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto build' + + build_script: su postgres -c 'ninja -C build' + upload_caches: ccache + + test_world_script: + - su postgres -c 'meson test --no-rebuild -C build' + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" + type: text/plain + + on_failure: + <<: *on_failure + cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores + + # configure feature flags, shared between the task running the linux tests and # the CompilerWarnings task LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >- @@ -141,14 +174,28 @@ LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >- --with-selinux --with-ssl=openssl --with-systemd - --with-tcl --with-tclconfig=/usr/lib/tcl8.6/ - --with-uuid=ossp + --with-tcl --with-tclconfig=${TCL_DIR:/usr/lib/tcl8.6/} + --with-uuid=${UUID:ossp} --with-zstd -task: - name: Linux - Debian Bullseye +# configure preparation scripts, shared between tasks running the linux tests +linux_preparation_scripts_template: &linux_preparation_scripts_template + sysinfo_script: | + id + uname -a + cat /proc/cmdline + ulimit -a -H && ulimit -a -S + export + create_user_script: | + useradd -m -U postgres + chown -R postgres:postgres . + mkdir -p ${CCACHE_DIR} + chown -R postgres:postgres ${CCACHE_DIR} + su postgres -c "ulimit -l -H && ulimit -l -S" + +linux_debian_template: &linux_debian_template env: CPUS: 4 BUILD_JOBS: 4 @@ -171,19 +218,9 @@ task: ccache_cache: folder: ${CCACHE_DIR} - sysinfo_script: | - id - uname -a - cat /proc/cmdline - ulimit -a -H && ulimit -a -S - export - create_user_script: | - useradd -m postgres - chown -R postgres:postgres . - mkdir -p ${CCACHE_DIR} - chown -R postgres:postgres ${CCACHE_DIR} + set_limits_script: | echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf - su postgres -c "ulimit -l -H && ulimit -l -S" + <<: *linux_preparation_scripts_template setup_core_files_script: | mkdir -m 770 /tmp/cores chown root:postgres /tmp/cores @@ -192,6 +229,12 @@ task: #apt-get update #DEBIAN_FRONTEND=noninteractive apt-get -y install ... + +task: + <<: *linux_debian_template + + name: Linux - Debian Bullseye - Autoconf + configure_script: | su postgres <<-EOF ./configure \ @@ -220,9 +263,351 @@ task: cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores -task: - name: macOS - Monterey +task: &linux_debian_bullseye_meson + <<: *linux_debian_template + name: Linux - Debian Bullseye - Meson + + configure_script: + - su postgres -c 'meson setup --buildtype debug -Dcassert=true -Dssl=openssl -Duuid=e2fs -Ddtrace=enabled build' + + build_script: su postgres -c 'ninja -C build' + upload_caches: ccache + + tests_world_script: + - su postgres -c 'meson test --no-rebuild -C build' + + on_failure: + <<: *on_failure + cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" + type: text/plain + + +task: + <<: *linux_debian_bullseye_meson + + name: Linux - Debian Sid - Meson + + compute_engine_instance: + image: family/pg-ci-sid + + +linux_opensuse_template: &linux_opensuse_template + env: + CPUS: 4 + BUILD_JOBS: 4 + TEST_JOBS: 8 # experimentally derived to be a decent choice + + CCACHE_DIR: /tmp/ccache_dir + DEBUGINFOD_URLS: "https://debuginfod.opensuse.org/" + + TCL_DIR: /usr/lib64/ + UUID: e2fs + LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES + + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + + container: + dockerfile: src/tools/ci/docker/linux_opensuse_tumbleweed + cpu: $CPUS + memory: 4G + + ccache_cache: + folder: ${CCACHE_DIR} + + <<: *linux_preparation_scripts_template + + +task: + <<: *linux_opensuse_template + + name: Linux - OpenSuse Tumbleweed - Autoconf + trigger_type: manual + + configure_script: | + su postgres <<-EOF + export PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" && \ + ./configure \ + --enable-cassert --enable-debug --enable-tap-tests \ + --enable-nls \ + \ + ${LINUX_CONFIGURE_FEATURES} + \ + CC="ccache gcc" \ + CXX="ccache g++" \ + CLANG="ccache clang" \ + CFLAGS="-O0 -ggdb" + EOF + build_script: su postgres -c 'make -s -j${BUILD_JOBS} world-bin' + upload_caches: ccache + + test_world_script: | + su postgres <<-EOF + ulimit -c unlimited; \ + make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + EOF + + on_failure: + <<: *on_failure + + +task: + <<: *linux_opensuse_template + + name: Linux - OpenSuse Tumbleweed (LLVM) - Meson + + configure_script: + - su postgres -c 'meson setup --buildtype debug -Dcassert=true -Dssl=openssl -Duuid=e2fs -Dllvm=enabled build' + + build_script: su postgres -c 'ninja -C build' + upload_caches: ccache + + tests_world_script: + - su postgres -c 'meson test --no-rebuild -C build --num-processes ${TEST_JOBS}' + + on_failure: + <<: *on_failure + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" + type: text/plain + + +linux_rhel_template: &linux_rhel_template + env: + CPUS: 4 + BUILD_JOBS: 4 + TEST_JOBS: 8 # experimentally derived to be a decent choice + + CCACHE_DIR: /tmp/ccache_dir + DEBUGINFOD_URLS: ${DEBUGINFO} + + TCL_DIR: /usr/lib64/ + UUID: e2fs + LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES + + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + + container: + dockerfile: ${DOCKERFILE_PATH} + cpu: $CPUS + memory: 4G + + ccache_cache: + folder: ${CCACHE_DIR} + + <<: *linux_preparation_scripts_template + + +task: + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_fedora_rawhide + DEBUGINFO: "https://debuginfod.fedoraproject.org/" + + <<: *linux_rhel_template + + name: Linux - Fedora Rawhide - Autoconf + trigger_type: manual + + configure_script: | + su postgres <<-EOF + ./configure \ + --enable-cassert --enable-debug --enable-tap-tests \ + --enable-nls \ + \ + ${LINUX_CONFIGURE_FEATURES} + \ + CC="ccache gcc" \ + CXX="ccache g++" \ + CLANG="ccache clang" \ + CFLAGS="-O0 -ggdb" + EOF + build_script: su postgres -c 'make -s -j${BUILD_JOBS} world-bin' + upload_caches: ccache + + test_world_script: | + su postgres <<-EOF + ulimit -c unlimited; \ + make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + EOF + + on_failure: + <<: *on_failure + + +task: + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_fedora_rawhide + DEBUGINFO: "https://debuginfod.fedoraproject.org/" + + <<: *linux_rhel_template + + name: Linux - Fedora Rawhide - Meson + + configure_script: + - su postgres -c 'meson setup --buildtype debug -Dcassert=true -Dssl=openssl -Duuid=e2fs build' + + build_script: su postgres -c 'ninja -C build' + upload_caches: ccache + + tests_world_script: + - su postgres -c 'meson test --no-rebuild -C build --num-processes ${TEST_JOBS}' + + on_failure: + <<: *on_failure + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" + type: text/plain + + +task: + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_centos8 + DEBUGINFO: "http://debuginfo.centos.org/" + + <<: *linux_rhel_template + + name: Linux - Centos 8 - Autoconf + trigger_type: manual + + configure_script: | + su postgres <<-EOF + ./configure \ + --enable-cassert --enable-debug --enable-tap-tests \ + --enable-nls \ + \ + ${LINUX_CONFIGURE_FEATURES} + \ + CC="ccache gcc" \ + CXX="ccache g++" \ + CLANG="ccache clang" \ + CFLAGS="-O0 -ggdb" + EOF + build_script: su postgres -c 'make -s -j${BUILD_JOBS} world-bin' + upload_caches: ccache + + test_world_script: | + su postgres <<-EOF + ulimit -c unlimited; \ + make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + EOF + + on_failure: + <<: *on_failure + + +task: + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_centos8 + DEBUGINFO: "http://debuginfo.centos.org/" + + <<: *linux_rhel_template + + name: Linux - Centos 8 - Meson + + configure_script: + - su postgres -c 'meson setup --buildtype debug -Dcassert=true -Dssl=openssl -Duuid=e2fs build' + + build_script: su postgres -c 'ninja -C build' + upload_caches: ccache + + tests_world_script: + - su postgres -c 'meson test --no-rebuild -C build --num-processes ${TEST_JOBS}' + + on_failure: + <<: *on_failure + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" + type: text/plain + + +task: + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_centos7 + DEBUGINFO: "http://debuginfo.centos.org/" + + <<: *linux_rhel_template + + name: Linux - Centos 7 - Autoconf + trigger_type: manual + + configure_script: | + su postgres <<-EOF + PYTHON=python3 \ + ./configure \ + --enable-cassert --enable-debug --enable-tap-tests \ + --enable-nls \ + \ + --with-gssapi \ + --with-icu \ + --with-ldap \ + --with-libxml \ + --with-libxslt \ + --with-lz4 \ + --with-pam \ + --with-perl \ + --with-python \ + --with-selinux \ + --with-ssl=openssl \ + --with-systemd \ + --with-tcl --with-tclconfig=/usr/lib64/ \ + --with-uuid=e2fs \ + \ + CC="ccache gcc" \ + CXX="ccache g++" \ + CLANG="ccache clang" \ + CFLAGS="-O0 -ggdb" + EOF + build_script: su postgres -c 'make -s -j${BUILD_JOBS} world-bin' + upload_caches: ccache + + test_world_script: | + su postgres <<-EOF + ulimit -c unlimited; \ + make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + EOF + + on_failure: + <<: *on_failure + + +task: + env: + DOCKERFILE_PATH: src/tools/ci/docker/linux_centos7 + DEBUGINFO: "http://debuginfo.centos.org/" + + <<: *linux_rhel_template + + name: Linux - Centos 7 - Meson + + configure_script: + - su postgres -c 'meson setup --buildtype debug -Dcassert=true -Dssl=openssl -Duuid=e2fs build' + + build_script: su postgres -c 'ninja -C build' + upload_caches: ccache + + tests_world_script: + - su postgres -c 'meson test --no-rebuild -C build --num-processes ${TEST_JOBS}' + + on_failure: + <<: *on_failure + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" + type: text/plain + + +macos_template: &macos_template env: CPUS: 12 # always get that much for cirrusci macOS instances BUILD_JOBS: $CPUS @@ -269,6 +654,7 @@ task: llvm \ lz4 \ make \ + meson \ openldap \ openssl \ python \ @@ -280,6 +666,14 @@ task: ccache_cache: folder: $CCACHE_DIR + + +task: + <<: *macos_template + + name: macOS - Monterey - Autoconf + trigger_type: manual + configure_script: | brewpath="/usr/local" INCLUDES="${brewpath}/include:${INCLUDES}" @@ -339,8 +733,42 @@ task: task: - name: Windows - Server 2019, VS 2019 + <<: *macos_template + name: macOS - Monterey - Meson + + configure_script: | + brewpath="/usr/local" + PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}" + + for pkg in icu4c krb5 openldap openssl ; do + pkgpath="${brewpath}/opt/${pkg}" + PKG_CONFIG_PATH="${pkgpath}/lib/pkgconfig:${PKG_CONFIG_PATH}" + done + + export PKG_CONFIG_PATH + + meson setup --buildtype debug -Dcassert=true -Dssl=openssl -Duuid=e2fs -Ddtrace=auto build + + build_script: ninja -C build + upload_caches: ccache + + tests_world_script: + - ulimit -c unlimited + - ulimit -n 1024 + - meson test --no-rebuild -C build + + on_failure: + <<: *on_failure + cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores" + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" + type: text/plain + + +windows_template: &windows_template env: # Half the allowed per-user CPU cores CPUS: 4 @@ -402,6 +830,11 @@ task: setup_additional_packages_script: | REM choco install -y --no-progress ... + +task: + <<: *windows_template + name: Windows - Server 2019, VS 2019 - Homegrown + configure_script: # copy errors out when using forward slashes - copy src\tools\ci\windows_build_config.pl src\tools\msvc\config.pl @@ -454,15 +887,81 @@ task: <<: *on_failure crashlog_artifacts: path: "crashlog-*.txt" + + +task: + <<: *windows_template + name: Windows - Server 2019, VS 2019 - Meson & ninja + + meson_install_script: + - pip install meson ninja + + configure_script: + - vcvarsall x64 + - mkdir subprojects + - meson wrap install lz4 + - meson wrap install zlib + - meson setup --buildtype debug --backend ninja -Dcassert=true -Db_pch=true -Dssl=openssl -Dlz4=enabled -Dzlib=enabled -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=c:/windows/system32/tar.exe build + + build_script: + - vcvarsall x64 + - ninja -C build + + check_world_script: + - vcvarsall x64 + - meson test --no-rebuild -C build + + on_failure: + <<: *on_failure + crashlog_artifacts: + path: "crashlog-*.txt" + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" type: text/plain +task: + <<: *windows_template + name: Windows - Server 2019, VS 2019 - Meson & msbuild + + meson_install_script: + - pip install meson + + configure_script: + - vcvarsall x64 + - mkdir subprojects + - meson wrap install lz4 + - meson wrap install zlib + - meson setup --buildtype debug --backend vs -Dcassert=true -Db_pch=true -Dssl=openssl -Dlz4=enabled -Dzlib=enabled -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=c:/windows/system32/tar.exe build + + build_script: + - vcvarsall x64 + - msbuild /p:UseMultiToolTask=true %MSBFLAGS% build\postgresql.sln + + check_world_script: + - vcvarsall x64 + - meson test --no-rebuild -C build + + always: + meson_log_artifacts: + path: "build/meson-logs/*.txt" + type: text/plain + + on_failure: + <<: *on_failure + crashlog_artifacts: + path: "crashlog-*.txt" + + task: name: CompilerWarnings # To limit unnecessary work only run this once the normal linux test succeeds depends_on: - - Linux - Debian Bullseye + - Linux - Debian Bullseye - Meson + trigger_type: manual env: CPUS: 4 diff --git a/src/tools/ci/docker/linux_centos7 b/src/tools/ci/docker/linux_centos7 new file mode 100644 index 00000000000..3742890d961 --- /dev/null +++ b/src/tools/ci/docker/linux_centos7 @@ -0,0 +1,57 @@ +FROM centos:centos7 +# Enable powertools, EPEL and devtoolset repository +RUN \ + yum -y install dnf-plugins-core \ + epel-release \ + centos-release-scl-rh && \ + \ + yum -y update && \ + yum -y install \ + \ + git \ + perl \ + perl-ExtUtils-Embed \ + perl-IPC-Run \ + perl-Test-Harness \ + perl-Test-Simple \ + \ + bison \ + ccache \ + clang \ + devtoolset-7-make \ + flex \ + gcc \ + gettext \ + \ + krb5-devel \ + libicu-devel \ + libuuid-devel \ + libxml2-devel \ + libxslt-devel \ + libzstd-devel \ + lz4-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + python3-devel \ + readline-devel \ + systemd-devel \ + tcl-devel \ + \ + krb5-server \ + krb5-server-ldap \ + krb5-workstation \ + lz4 \ + openldap-clients \ + openldap-servers \ + openssl \ + zstd \ + \ + && \ + pip3 install meson && \ + pip3 install ninja && \ + yum clean all + +# Make version is 3.82 but required is > 4, adding devtoolset-7-make to the PATH +ENV PATH="/opt/rh/devtoolset-7/root/usr/bin/:${PATH}" +ENV LANG=en_US.UTF-8 diff --git a/src/tools/ci/docker/linux_centos8 b/src/tools/ci/docker/linux_centos8 new file mode 100644 index 00000000000..58f95847ef1 --- /dev/null +++ b/src/tools/ci/docker/linux_centos8 @@ -0,0 +1,54 @@ +FROM quay.io/centos/centos:stream8 +# Enable powertools and EPEL repository +RUN \ + dnf -y install dnf-plugins-core && \ + dnf config-manager --set-enabled powertools && \ + dnf -y install epel-release && \ + dnf -y update && \ + dnf -y install \ + \ + git \ + meson \ + perl \ + perl-IPC-Run \ + \ + bison \ + ccache \ + clang \ + diffutils \ + flex \ + gcc \ + gettext \ + \ + krb5-devel \ + libicu-devel \ + libuuid-devel \ + libxml2-devel \ + libxslt-devel \ + libzstd-devel \ + llvm-devel \ + lz4-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + python3-devel \ + readline-devel \ + systemd-devel \ + tcl-devel \ + \ + krb5-server \ + krb5-server-ldap \ + krb5-workstation \ + lz4 \ + openldap-clients \ + openldap-servers \ + openssl \ + zstd \ + \ + glibc-langpack-en \ + \ + && \ + pip3 install ninja && \ + yum clean all + +ENV LANG=en_US.UTF-8 diff --git a/src/tools/ci/docker/linux_fedora_rawhide b/src/tools/ci/docker/linux_fedora_rawhide new file mode 100644 index 00000000000..c95027cd6a7 --- /dev/null +++ b/src/tools/ci/docker/linux_fedora_rawhide @@ -0,0 +1,49 @@ +FROM fedora:rawhide +RUN \ + dnf -y update && \ + dnf -y install \ + git \ + meson \ + perl \ + perl-IPC-Run \ + \ + bison \ + ccache \ + clang \ + flex \ + gcc \ + gettext \ + ninja-build \ + \ + krb5-devel \ + libicu-devel \ + libuuid-devel \ + libxml2-devel \ + libxslt-devel \ + libzstd-devel \ + llvm-devel \ + lz4-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + python3-devel \ + readline-devel \ + systemd-devel \ + tcl-devel \ + \ + krb5-server \ + krb5-server-ldap \ + krb5-workstation \ + lz4 \ + openldap-clients \ + openldap-servers \ + openssl \ + zstd \ + \ + langpacks-en \ + glibc-langpack-en \ + \ + && \ + yum clean all + +ENV LANG=en_US.UTF-8 diff --git a/src/tools/ci/docker/linux_opensuse_tumbleweed b/src/tools/ci/docker/linux_opensuse_tumbleweed new file mode 100644 index 00000000000..07364ee85ff --- /dev/null +++ b/src/tools/ci/docker/linux_opensuse_tumbleweed @@ -0,0 +1,56 @@ +FROM opensuse/tumbleweed +RUN \ + zypper -n clean -a && \ + zypper -n ref && \ + zypper -n dup && \ + zypper -n update && \ + zypper -n install \ + diffutils \ + git \ + icu \ + make \ + meson \ + perl-IPC-Run \ + shadow \ + systemd-devel \ + util-linux \ + \ + bison \ + ccache \ + clang \ + flex \ + gcc \ + gcc-c++ \ + gettext-runtime \ + ninja \ + \ + krb5-devel \ + libicu-devel \ + libldapcpp-devel \ + liblz4-devel \ + libopenssl-devel \ + libselinux-devel \ + libuuid-devel \ + libxml2-devel \ + libxslt-devel \ + libzstd-devel \ + llvm-devel \ + pam-devel \ + python38-devel \ + readline-devel \ + tcl-devel \ + \ + openldap2 \ + openldap2-client \ + \ + krb5-client \ + krb5-plugin-kdb-ldap \ + krb5-server \ + lz4 \ + zstd \ + \ + && \ + zypper -n clean -a + +# Fixing ICU errors caused by locale being set to 'POSIX' +ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 -- 2.37.0.3.g30cc8d0f14