From 00c48a90f3acc6cba6af873b29429ee6d4ba38a6 Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Thu, 3 Aug 2023 23:29:13 -0700
Subject: [PATCH v1 1/9] ci: macos: used cached macports install

This substantially speeds up the mac CI time.

Discussion: https://postgr.es/m/20230805202539.r3umyamsnctysdc7@awork3.anarazel.de
---
 .cirrus.yml                          | 63 +++++++-----------
 src/tools/ci/ci_macports_packages.sh | 97 ++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+), 38 deletions(-)
 create mode 100755 src/tools/ci/ci_macports_packages.sh

diff --git a/.cirrus.yml b/.cirrus.yml
index d260f15c4e2..e9cfc542cfe 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -429,8 +429,7 @@ task:
 
     CIRRUS_WORKING_DIR: ${HOME}/pgsql/
     CCACHE_DIR: ${HOME}/ccache
-    HOMEBREW_CACHE: ${HOME}/homebrew-cache
-    PERL5LIB: ${HOME}/perl5/lib/perl5
+    MACPORTS_CACHE: ${HOME}/macports-cache
 
     CC: ccache cc
     CXX: ccache c++
@@ -454,55 +453,43 @@ task:
     - mkdir ${HOME}/cores
     - sudo sysctl kern.corefile="${HOME}/cores/core.%P"
 
-  perl_cache:
-    folder: ~/perl5
-  cpan_install_script:
-    - perl -mIPC::Run -e 1 || cpan -T IPC::Run
-    - perl -mIO::Pty -e 1 || cpan -T IO::Pty
-  upload_caches: perl
-
-
-  # XXX: Could we instead install homebrew into a cached directory? The
-  # homebrew installation takes a good bit of time every time, even if the
-  # packages do not need to be downloaded.
-  homebrew_cache:
-    folder: $HOMEBREW_CACHE
+  # Use macports, even though homebrew is installed. The installation
+  # of the additional packages we need would take quite a while with
+  # homebrew, even if we cache the downloads. We can't cache all of
+  # homebrew, because it's already large. So we use macports. To cache
+  # the installation we create a .dmg file that we mount if it already
+  # exists.
+  # XXX: The reason for the direct p5.34* references is that we'd need
+  # the large macport tree around to figure out that p5-io-tty is
+  # actually p5.34-io-tty. Using the unversioned name works, but
+  # updates macports every time.
+  macports_cache:
+    folder: ${MACPORTS_CACHE}
   setup_additional_packages_script: |
-    brew install \
+    sh src/tools/ci/ci_macports_packages.sh \
       ccache \
-      icu4c \
-      krb5 \
-      llvm \
+      icu \
+      kerberos5 \
       lz4 \
-      make \
       meson \
       openldap \
       openssl \
-      python \
-      tcl-tk \
+      p5.34-io-tty \
+      p5.34-ipc-run \
+      tcl \
       zstd
-
-    brew cleanup -s # to reduce cache size
-  upload_caches: homebrew
+    # Make macports install visible for subsequent steps
+    echo PATH=/opt/local/sbin/:/opt/local/bin/:$PATH >> $CIRRUS_ENV
+  upload_caches: macports
 
   ccache_cache:
     folder: $CCACHE_DIR
   configure_script: |
-    brewpath="/opt/homebrew"
-    PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
-
-    for pkg in icu4c krb5 openldap openssl zstd ; do
-      pkgpath="${brewpath}/opt/${pkg}"
-      PKG_CONFIG_PATH="${pkgpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
-      PATH="${pkgpath}/bin:${pkgpath}/sbin:$PATH"
-    done
-
-    export PKG_CONFIG_PATH PATH
-
+    export PKG_CONFIG_PATH="/opt/local/lib/pkgconfig/"
     meson setup \
       --buildtype=debug \
-      -Dextra_include_dirs=${brewpath}/include \
-      -Dextra_lib_dirs=${brewpath}/lib \
+      -Dextra_include_dirs=/opt/local/include \
+      -Dextra_lib_dirs=/opt/local/lib \
       -Dcassert=true \
       -Duuid=e2fs -Ddtrace=auto \
       -Dsegsize_blocks=6 \
diff --git a/src/tools/ci/ci_macports_packages.sh b/src/tools/ci/ci_macports_packages.sh
new file mode 100755
index 00000000000..5f5d3027760
--- /dev/null
+++ b/src/tools/ci/ci_macports_packages.sh
@@ -0,0 +1,97 @@
+#!/bin/sh
+
+# Installs the passed in packages via macports. To make it fast enough
+# for CI, cache the installation as a .dmg file.  To avoid
+# unnecessarily updating the cache, the cached image is only modified
+# when packages are installed or removed.  Any package this script is
+# not instructed to install, will be removed again.
+#
+# This currently expects to be run in a macos cirrus-ci environment.
+
+set -e
+set -x
+
+packages="$@"
+
+macports_url="https://github.com/macports/macports-base/releases/download/v2.8.1/MacPorts-2.8.1-13-Ventura.pkg"
+cache_dmg="macports.hfs.dmg"
+
+if [ "$CIRRUS_CI" != "true" ]; then
+    echo "expect to be called within cirrus-ci" 1>2
+    exit 1
+fi
+
+sudo mkdir -p /opt/local
+mkdir -p ${MACPORTS_CACHE}/
+
+# If we are starting from clean cache, perform a fresh macports
+# install. Otherwise decompress the .dmg we created previously.
+#
+# After this we have a working macports installation, with an unknown set of
+# packages installed.
+new_install=0
+update_cached_image=0
+if [ -e ${MACPORTS_CACHE}/${cache_dmg}.zstd ]; then
+    time zstd -T0 -d ${MACPORTS_CACHE}/${cache_dmg}.zstd -o ${cache_dmg}
+    time sudo hdiutil attach -kernel ${cache_dmg} -owners on -shadow ${cache_dmg}.shadow -mountpoint /opt/local
+else
+    new_install=1
+    curl -fsSL -o macports.pkg "$macports_url"
+    time sudo installer -pkg macports.pkg -target /
+    # this is a throwaway environment, and it'd be a few lines to gin
+    # up a correct user / group when using the cache.
+    echo macportsuser root | sudo tee -a /opt/local/etc/macports/macports.conf
+fi
+export PATH=/opt/local/sbin/:/opt/local/bin/:$PATH
+
+# mark all installed packages unrequested, that allows us to detect
+# packages that aren't needed anymore
+if [ -n "$(port -q installed installed)" ] ; then
+    sudo port unsetrequested installed
+fi
+
+# if setting all the required packages as requested fails, we need
+# to install at least one of them
+if ! sudo port setrequested $packages > /dev/null 2>&1 ; then
+    echo not all required packages installed, doing so now
+    update_cached_image=1
+    # to keep the image small, we deleted the ports tree from the image...
+    sudo port selfupdate
+    # XXX likely we'll need some other way to force an upgrade at some
+    # point...
+    sudo port upgrade outdated
+    sudo port install -N $packages
+    sudo port setrequested $packages
+fi
+
+# check if any ports should be uninstalled
+if [ -n "$(port -q installed rleaves)" ] ; then
+    echo superflous packages installed
+    update_cached_image=1
+    sudo port uninstall --follow-dependencies rleaves
+
+    # remove prior cache contents, don't want to increase size
+    rm -f ${MACPORTS_CACHE}/*
+fi
+
+# Shrink installation if we created / modified it
+if [ "$new_install" -eq 1 -o "$update_cached_image" -eq 1 ]; then
+    sudo /opt/local/bin/port clean --all installed
+    sudo rm -rf /opt/local/var/macports/{software,sources}/*
+fi
+
+# If we're starting from a clean cache, start a new image. If we have
+# an image, but the contents changed, update the image in the cache
+# location.
+if [ "$new_install" -eq 1 ]; then
+    # use a generous size, so additional software can be installed later
+    time sudo hdiutil create -fs HFS+ -format UDRO -size 10g -layout NONE -srcfolder /opt/local/ ${cache_dmg}
+    time zstd -T -10 -z ${cache_dmg} -o ${MACPORTS_CACHE}/${cache_dmg}.zstd
+elif [ "$update_cached_image" -eq 1 ]; then
+    sudo hdiutil detach /opt/local/
+    time hdiutil convert -format UDRO ${cache_dmg} -shadow ${cache_dmg}.shadow -o updated.hfs.dmg
+    rm ${cache_dmg}.shadow
+    mv updated.hfs.dmg ${cache_dmg}
+    time zstd -T -10 -z ${cache_dmg} -o ${MACPORTS_CACHE}/${cache_dmg}.zstd
+    time sudo hdiutil attach -kernel ${cache_dmg} -owners on -shadow ${cache_dmg}.shadow -mountpoint /opt/local
+fi
-- 
2.38.0