From b22ea942f5c23530c2634d1224c1a2e8a7468478 Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Fri, 7 Mar 2025 12:10:58 +0300 Subject: [PATCH v13 3/4] meson: Add architecture for LLVM bitcode emission This commit adds support for bitcode emission for backend and extension source files. These bitcode files are installed into the $pkglibdir/bitcode/ directory if LLVM is found. New variable `bitcode_modules` is introduced to generate bitcode files. All required information is gathered in this variable. Then, this variable is processed by the main meson LLVM bitcode emission scripts: src/backend/jit/llvm/bitcode/meson.build -> src/tools/irlink. An example of a possible structure of bitcode_modules is: ``` bitcode_modules = [ { 'name': '...', 'target': ..., 'srcfiles': [ '...', '...', ], 'additional_flags': [ '-I...', '-I...', ], } ] ``` Instead of depending on the generated backend headers explicitly, each bitcode target lists the corresponding extracted object file as an input. The command only consumes the source file, but referencing the object file pulls in its dependencies (e.g. generated headers) without the large slowdown in build.ninja generation that an explicit depends causes. Author: Andres Freund Author: Nazir Bilal Yavuz Author: Diego Fronza Reviewed-by: Peter Eisentraut Reviewed-by: Diego Fronza Reviewed-by: Zsolt Parragi Discussion: https://postgr.es/m/206b001d-1884-4081-bd02-bed5c92f02ba%40eisentraut.org --- meson.build | 21 +++++++++++ src/backend/jit/llvm/bitcode/meson.build | 47 ++++++++++++++++++++++++ src/backend/jit/llvm/meson.build | 40 +++++++++++++------- src/backend/meson.build | 9 +++++ src/makefiles/meson.build | 5 +-- src/tools/irlink | 28 ++++++++++++++ 6 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 src/backend/jit/llvm/bitcode/meson.build create mode 100644 src/tools/irlink diff --git a/meson.build b/meson.build index d88a7a70308..738422a97ec 100644 --- a/meson.build +++ b/meson.build @@ -937,6 +937,8 @@ if have_cxx # Some distros put LLVM and clang in different paths, so fallback to # find via PATH, too. clang = find_program(llvm_binpath / 'clang', 'clang', required: true) + llvm_lto = find_program(llvm_binpath / 'llvm-lto', required: true) + irlink = find_program('src/tools/irlink', native: true) endif else msg = 'llvm requires a C++ compiler' @@ -3381,6 +3383,11 @@ update_unicode_targets = [] test_deps = [] tests = [] +# List of object files + source files to generated LLVM IR for inlining. +# Each element is a hash of: +# {'target': target, 'srcfiles': ..., 'additional_flags': ...}. +bitcode_modules = [] + # Default options for targets @@ -3704,6 +3711,11 @@ subdir('src/interfaces/ecpg/test') subdir('doc/src/sgml') +# generate bitcode for JIT inlining after giving contrib modules etc a chance +# to add themselves to bitcode_modules[] +subdir('src/backend/jit/llvm/bitcode', if_found: llvm) + + generated_sources_ac += {'': ['GNUmakefile']} # After processing src/test, add test_install_libs to the testprep_targets @@ -4314,6 +4326,15 @@ summary( section: 'Programs', ) +if llvm.found() + summary( + { + 'clang': clang, + }, + section: 'Programs', + ) +endif + summary( { 'bonjour': bonjour, diff --git a/src/backend/jit/llvm/bitcode/meson.build b/src/backend/jit/llvm/bitcode/meson.build new file mode 100644 index 00000000000..8c0448ea6c6 --- /dev/null +++ b/src/backend/jit/llvm/bitcode/meson.build @@ -0,0 +1,47 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group +# +# emit LLVM bitcode for JIT inlining + +assert(llvm.found()) + +foreach bitcode_module : bitcode_modules + bitcode_targets = [] + bitcode_obj = bitcode_module['target'] + bitcode_cflags_local = bitcode_cflags + bitcode_module.get('additional_flags', []) + bitcode_name = bitcode_module.get('name', bitcode_obj.name()) + + foreach srcfile : bitcode_module['srcfiles'] + if meson.version().version_compare('>=0.59') + srcfilename = fs.parent(srcfile) / fs.name(srcfile) + else + srcfilename = '@0@'.format(srcfile) + endif + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + # Depend on the bitcode object file's extracted object. Referencing the + # object file pulls in its dependencies (e.g. generated headers) without + # slowing down the build. + input: [srcfile, bitcode_obj.extract_objects(srcfile)], + output: targetname, + command: [llvm_irgen_command, llvm_irgen_dep_args, bitcode_cflags_local], + depfile: targetname + '.d', + install: true, + install_dir: dir_bitcode, + ) + endforeach + + index_name = '@0@.index.bc'.format(bitcode_name) + bitcode_index = custom_target('@0@'.format(bitcode_name), + output: index_name, + input: bitcode_targets, + command: [irlink, '--lto', llvm_lto, '--outdir', '@OUTDIR@', '--index', index_name, '@INPUT@'], + install: true, + install_dir: dir_bitcode, + ) + backend_targets += bitcode_index +endforeach diff --git a/src/backend/jit/llvm/meson.build b/src/backend/jit/llvm/meson.build index 7df8453ad6f..c3b5464f8c1 100644 --- a/src/backend/jit/llvm/meson.build +++ b/src/backend/jit/llvm/meson.build @@ -42,28 +42,32 @@ backend_targets += llvmjit # Define a few bits and pieces used here and elsewhere to generate bitcode -llvm_irgen_args = [ - '-c', '-o', '@OUTPUT@', '@INPUT@', +llvm_irgen_command = [] +if ccache.found() + llvm_irgen_command += ccache +endif + +llvm_irgen_command += [ + clang, + '-c', '-o', '@OUTPUT0@', '@INPUT0@', '-flto=thin', '-emit-llvm', - '-MD', '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@', - '-O2', '-Wno-ignored-attributes', '-Wno-empty-body', + '-Wno-unknown-warning-option', + '-Wno-compound-token-split-by-macro', ] - -if ccache.found() - llvm_irgen_command = ccache - llvm_irgen_args = [clang.full_path()] + llvm_irgen_args -else - llvm_irgen_command = clang -endif +llvm_irgen_dep_args = ['-MD', '-MQ', '@OUTPUT0@', '-MF', '@DEPFILE@'] # XXX: Need to determine proper version of the function cflags for clang -bitcode_cflags = ['-fno-strict-aliasing', '-fwrapv'] -bitcode_cflags += get_option('c_args') +bitcode_cflags = ['-fno-strict-aliasing', '-fwrapv', '-O2'] bitcode_cflags += cppflags +var_bitcode_cxxflags = bitcode_cflags +var_bitcode_cxxflags += get_option('cpp_args') +bitcode_cflags += get_option('c_args') +var_bitcode_cflags = bitcode_cflags + # XXX: Worth improving on the logic to find directories here bitcode_cflags += '-I@BUILD_ROOT@/src/include' bitcode_cflags += '-I@BUILD_ROOT@/src/backend/utils/misc' @@ -73,7 +77,7 @@ bitcode_cflags += '-I@SOURCE_ROOT@/src/include' # Note this is intentionally not installed to bitcodedir, as it's not for # inlining llvmjit_types = custom_target('llvmjit_types.bc', - command: [llvm_irgen_command] + llvm_irgen_args + bitcode_cflags, + command: llvm_irgen_command + llvm_irgen_dep_args + bitcode_cflags, input: 'llvmjit_types.c', output: 'llvmjit_types.bc', depends: [postgres], @@ -82,3 +86,11 @@ llvmjit_types = custom_target('llvmjit_types.bc', depfile: '@BASENAME@.c.bc.d', ) backend_targets += llvmjit_types + +# Figure out -I's needed to build all postgres code, including all its +# dependencies +pkg_config = find_program(['pkg-config', 'pkgconf'], required: true) +r = run_command(pkg_config, + ['--cflags-only-I', meson.project_build_root() / 'meson-uninstalled/postgresql-@0@-llvm-jit-bitcode-uninstalled.pc'.format(pg_version_major)], + check: true) +bitcode_cflags += r.stdout().split() diff --git a/src/backend/meson.build b/src/backend/meson.build index 759b53737b2..2b53265dafe 100644 --- a/src/backend/meson.build +++ b/src/backend/meson.build @@ -7,6 +7,9 @@ backend_link_with = [pgport_srv, common_srv] generated_backend_sources = [] post_export_backend_sources = [] +var_bitcode_cflags = [] +var_bitcode_cxxflags = [] + subdir('access') subdir('archive') subdir('backup') @@ -165,6 +168,12 @@ postgres = executable('postgres', backend_targets += postgres +bitcode_modules += { + 'name': 'postgres', + 'target': postgres_lib, + 'srcfiles': backend_sources, +} + pg_mod_c_args = cflags_mod pg_mod_cxx_args = cxxflags_mod pg_mod_link_args = ldflags_sl + ldflags_mod diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build index 2401025d1cd..1d09aed1d86 100644 --- a/src/makefiles/meson.build +++ b/src/makefiles/meson.build @@ -113,9 +113,8 @@ pgxs_kv = { ' '.join(cc.get_supported_link_arguments('-Wl,--export-dynamic')), 'LDFLAGS_SL': var_ldflags_sl, - # TODO: requires bitcode generation to be implemented for meson - 'BITCODE_CFLAGS': '', - 'BITCODE_CXXFLAGS': '', + 'BITCODE_CFLAGS': ' '.join(var_bitcode_cflags), + 'BITCODE_CXXFLAGS': ' '.join(var_bitcode_cxxflags), 'BISONFLAGS': ' '.join(bison_flags), 'FLEXFLAGS': ' '.join(flex_flags), diff --git a/src/tools/irlink b/src/tools/irlink new file mode 100644 index 00000000000..6a03e4f5695 --- /dev/null +++ b/src/tools/irlink @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Link per-source LLVM bitcode files into a single ThinLTO index module +# (e.g. postgres.index.bc) used for JIT inlining. + +import argparse +import os +import shutil +import subprocess +import sys + +parser = argparse.ArgumentParser( + description='generate PostgreSQL JIT IR module') + +parser.add_argument('--index', type=str, required=True) +parser.add_argument('--lto', type=str, required=True) +parser.add_argument('--outdir', type=str, required=True) +parser.add_argument('INPUT', type=str, nargs='+') + +args = parser.parse_args() + +file_names = [os.path.basename(f) for f in args.INPUT] +command = [args.lto, + '-thinlto', '-thinlto-action=thinlink', + '-o', args.index] + file_names +res = subprocess.run(command, cwd=args.outdir) + +exit(res.returncode) -- 2.47.3