From 79606cf7492eaa4b6a233179133da290e1a1db3b Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Fri, 7 Mar 2025 12:10:58 +0300 Subject: [PATCH v16 3/6] meson: Add architecture for LLVM bitcode emission This commit adds support for bitcode emission for backend and extension source files. These bitcode files are installed into the $pkglibdir/bitcode/ directory if LLVM is found. New variable `bitcode_modules` is introduced to generate bitcode files. All required information is gathered in this variable. Then, this variable is processed by the main meson LLVM bitcode emission scripts: src/backend/jit/llvm/bitcode/meson.build -> src/tools/irlink. An example of a possible structure of bitcode_modules is: ``` bitcode_modules = [ { 'name': '...', 'target': ..., 'srcfiles': [ '...', '...', ], 'additional_flags': [ '-I...', '-I...', ], } ] ``` Instead of depending on the generated backend headers explicitly, each bitcode target lists the corresponding extracted object file as an input. The command only consumes the source file, but referencing the object file pulls in its dependencies (e.g. generated headers) without the large slowdown in build.ninja generation that an explicit depends causes. Author: Andres Freund Author: Nazir Bilal Yavuz Author: Diego Fronza Reviewed-by: Peter Eisentraut Reviewed-by: Diego Fronza Reviewed-by: Zsolt Parragi Discussion: https://postgr.es/m/206b001d-1884-4081-bd02-bed5c92f02ba%40eisentraut.org --- meson.build | 21 +++++++++++ src/backend/jit/llvm/bitcode/meson.build | 47 ++++++++++++++++++++++++ src/backend/jit/llvm/meson.build | 40 +++++++++++++------- src/backend/meson.build | 9 +++++ src/makefiles/meson.build | 5 +-- src/tools/irlink | 26 +++++++++++++ 6 files changed, 131 insertions(+), 17 deletions(-) create mode 100644 src/backend/jit/llvm/bitcode/meson.build create mode 100755 src/tools/irlink diff --git a/meson.build b/meson.build index 3595c847d11..48de7aa8c20 100644 --- a/meson.build +++ b/meson.build @@ -937,6 +937,8 @@ if have_cxx # Some distros put LLVM and clang in different paths, so fallback to # find via PATH, too. clang = find_program(llvm_binpath / 'clang', 'clang', required: true) + llvm_lto = find_program(llvm_binpath / 'llvm-lto', required: true) + irlink = find_program('src/tools/irlink', native: true) endif else msg = 'llvm requires a C++ compiler' @@ -3379,6 +3381,11 @@ update_unicode_targets = [] test_deps = [] tests = [] +# List of object files + source files to generated LLVM IR for inlining. +# Each element is a hash of: +# {'target': target, 'srcfiles': ..., 'additional_flags': ...}. +bitcode_modules = [] + # Default options for targets @@ -3702,6 +3709,11 @@ subdir('src/interfaces/ecpg/test') subdir('doc/src/sgml') +# generate bitcode for JIT inlining after giving contrib modules etc a chance +# to add themselves to bitcode_modules[] +subdir('src/backend/jit/llvm/bitcode', if_found: llvm) + + generated_sources_ac += {'': ['GNUmakefile']} # After processing src/test, add test_install_libs to the testprep_targets @@ -4341,6 +4353,15 @@ summary( section: 'Programs', ) +if llvm.found() + summary( + { + 'clang': clang, + }, + section: 'Programs', + ) +endif + summary( { 'bonjour': bonjour, diff --git a/src/backend/jit/llvm/bitcode/meson.build b/src/backend/jit/llvm/bitcode/meson.build new file mode 100644 index 00000000000..8c0448ea6c6 --- /dev/null +++ b/src/backend/jit/llvm/bitcode/meson.build @@ -0,0 +1,47 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group +# +# emit LLVM bitcode for JIT inlining + +assert(llvm.found()) + +foreach bitcode_module : bitcode_modules + bitcode_targets = [] + bitcode_obj = bitcode_module['target'] + bitcode_cflags_local = bitcode_cflags + bitcode_module.get('additional_flags', []) + bitcode_name = bitcode_module.get('name', bitcode_obj.name()) + + foreach srcfile : bitcode_module['srcfiles'] + if meson.version().version_compare('>=0.59') + srcfilename = fs.parent(srcfile) / fs.name(srcfile) + else + srcfilename = '@0@'.format(srcfile) + endif + + targetname = '@0@_@1@.bc'.format( + bitcode_name, + srcfilename.underscorify(), + ) + bitcode_targets += custom_target( + targetname, + # Depend on the bitcode object file's extracted object. Referencing the + # object file pulls in its dependencies (e.g. generated headers) without + # slowing down the build. + input: [srcfile, bitcode_obj.extract_objects(srcfile)], + output: targetname, + command: [llvm_irgen_command, llvm_irgen_dep_args, bitcode_cflags_local], + depfile: targetname + '.d', + install: true, + install_dir: dir_bitcode, + ) + endforeach + + index_name = '@0@.index.bc'.format(bitcode_name) + bitcode_index = custom_target('@0@'.format(bitcode_name), + output: index_name, + input: bitcode_targets, + command: [irlink, '--lto', llvm_lto, '--outdir', '@OUTDIR@', '--index', index_name, '@INPUT@'], + install: true, + install_dir: dir_bitcode, + ) + backend_targets += bitcode_index +endforeach diff --git a/src/backend/jit/llvm/meson.build b/src/backend/jit/llvm/meson.build index dc2e129faf1..517e41d0bd2 100644 --- a/src/backend/jit/llvm/meson.build +++ b/src/backend/jit/llvm/meson.build @@ -42,28 +42,32 @@ backend_targets += llvmjit # Define a few bits and pieces used here and elsewhere to generate bitcode -llvm_irgen_args = [ - '-c', '-o', '@OUTPUT@', '@INPUT@', +llvm_irgen_command = [] +if ccache.found() + llvm_irgen_command += ccache +endif + +llvm_irgen_command += [ + clang, + '-c', '-o', '@OUTPUT0@', '@INPUT0@', '-flto=thin', '-emit-llvm', - '-MD', '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@', - '-O2', '-Wno-ignored-attributes', '-Wno-empty-body', + '-Wno-unknown-warning-option', + '-Wno-compound-token-split-by-macro', ] - -if ccache.found() - llvm_irgen_command = ccache - llvm_irgen_args = [clang.full_path()] + llvm_irgen_args -else - llvm_irgen_command = clang -endif +llvm_irgen_dep_args = ['-MD', '-MQ', '@OUTPUT0@', '-MF', '@DEPFILE@'] # XXX: Need to determine proper version of the function cflags for clang -bitcode_cflags = ['-fno-strict-aliasing', '-fwrapv'] -bitcode_cflags += get_option('c_args') +bitcode_cflags = ['-fno-strict-aliasing', '-fwrapv', '-O2'] bitcode_cflags += cppflags +var_bitcode_cxxflags = bitcode_cflags +var_bitcode_cxxflags += get_option('cpp_args') +bitcode_cflags += get_option('c_args') +var_bitcode_cflags = bitcode_cflags + # XXX: Worth improving on the logic to find directories here bitcode_cflags += '-I@BUILD_ROOT@/src/include' bitcode_cflags += '-I@BUILD_ROOT@/src/backend/utils/misc' @@ -73,7 +77,7 @@ bitcode_cflags += '-I@SOURCE_ROOT@/src/include' # Note this is intentionally not installed to bitcodedir, as it's not for # inlining llvmjit_types = custom_target('llvmjit_types.bc', - command: [llvm_irgen_command] + llvm_irgen_args + bitcode_cflags, + command: llvm_irgen_command + llvm_irgen_dep_args + bitcode_cflags, input: 'llvmjit_types.c', output: 'llvmjit_types.bc', depends: [postgres], @@ -82,3 +86,11 @@ llvmjit_types = custom_target('llvmjit_types.bc', depfile: '@BASENAME@.c.bc.d', ) backend_targets += llvmjit_types + +# Figure out -I's needed to build all postgres code, including all its +# dependencies +pkg_config = find_program(['pkg-config', 'pkgconf'], required: true) +r = run_command(pkg_config, + ['--cflags-only-I', meson.project_build_root() / 'meson-uninstalled/postgresql-@0@-llvm-jit-bitcode-uninstalled.pc'.format(pg_version_major)], + check: true) +bitcode_cflags += r.stdout().split() diff --git a/src/backend/meson.build b/src/backend/meson.build index 94b2418ed53..f6f41dc144c 100644 --- a/src/backend/meson.build +++ b/src/backend/meson.build @@ -7,6 +7,9 @@ backend_link_with = [pgport_srv, common_srv] generated_backend_sources = [] post_export_backend_sources = [] +var_bitcode_cflags = [] +var_bitcode_cxxflags = [] + subdir('access') subdir('archive') subdir('backup') @@ -165,6 +168,12 @@ postgres = executable('postgres', backend_targets += postgres +bitcode_modules += { + 'name': 'postgres', + 'target': postgres_lib, + 'srcfiles': backend_sources, +} + pg_mod_c_args = cflags_mod pg_mod_cxx_args = cxxflags_mod pg_mod_link_args = ldflags_sl + ldflags_mod diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build index d0430b3519e..4073586a09c 100644 --- a/src/makefiles/meson.build +++ b/src/makefiles/meson.build @@ -115,9 +115,8 @@ pgxs_kv = { ' '.join(cc.get_supported_link_arguments('-Wl,--export-dynamic')), 'LDFLAGS_SL': var_ldflags_sl, - # TODO: requires bitcode generation to be implemented for meson - 'BITCODE_CFLAGS': '', - 'BITCODE_CXXFLAGS': '', + 'BITCODE_CFLAGS': ' '.join(var_bitcode_cflags), + 'BITCODE_CXXFLAGS': ' '.join(var_bitcode_cxxflags), 'BISONFLAGS': ' '.join(bison_flags), 'FLEXFLAGS': ' '.join(flex_flags), diff --git a/src/tools/irlink b/src/tools/irlink new file mode 100755 index 00000000000..d3e30cc0581 --- /dev/null +++ b/src/tools/irlink @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# +# Link per-source LLVM bitcode files into a single ThinLTO index module +# (e.g. postgres.index.bc) used for JIT inlining. + +import argparse +import os +import subprocess + +parser = argparse.ArgumentParser( + description='generate PostgreSQL JIT IR module') + +parser.add_argument('--index', type=str, required=True) +parser.add_argument('--lto', type=str, required=True) +parser.add_argument('--outdir', type=str, required=True) +parser.add_argument('INPUT', type=str, nargs='+') + +args = parser.parse_args() + +file_names = [os.path.basename(f) for f in args.INPUT] +command = [args.lto, + '-thinlto', '-thinlto-action=thinlink', + '-o', args.index] + file_names +res = subprocess.run(command, cwd=args.outdir) + +exit(res.returncode) -- 2.47.3