From b9101ab942bb8f954450fc5a2b2b72168c370ce1 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Tue, 3 Mar 2026 20:23:55 -0500 Subject: [PATCH v5 1/5] aio: io_uring: Trigger async processing for large IOs io_method=io_uring has a heuristic to trigger asynchronous processing of IOs once the IO depth is a bit larger. That heuristic is important when doing buffered IO from the kernel page cache, to allow parallelizing of the memory copy, as otherwise io_method=io_uring would be a lot slower than io_method=worker in that case. An upcoming commit will make read_stream.c only increase the read-ahead distance if we needed to wait for IO to complete. If to-be-read data is in the kernel page cache, io_uring will synchronously execute IO, unless the IO is flagged as async. Therefore the aforementioned change in read_stream.c heuristic would lead to a substantial performance regression with io_uring when data is in the page cache, as we would never reach a deep enough queue to actually trigger the existing heuristic. Parallelizing the copy from the page cache is mainly important when doing a lot of IO, which commonly is only possible when doing largely sequential IO. The reason we don't just mark all io_uring IOs as asynchronous is that the dispatch to a kernel thread has overhead. This overhead is mostly noticeable with small random IOs with a low queue depth, as in that case the gain from parallelizing the memory copy is small and the latency cost high. The facts from the two prior paragraphs show a way out: Use the size of the IO in addition to the depth of the queue to trigger asynchronous processing. One might think that just using the IO size might be enough, but experimentation has shown that not to be the case - with deep look-ahead distances being able to parallelize the memory copy is important even with smaller IOs. Reviewed-by: Melanie Plageman Reviewed-by: Nazir Bilal Yavuz Discussion: https://postgr.es/m/f3xxfrkafjxpyqxywcxricxgyizjirfceychyxsgn7bwjp5eda@kwbduhy7tfmu Discussion: https://postgr.es/m/CA+hUKGL2PhFyDoqrHefqasOnaXhSg48t1phs3VM8BAdrZqKZkw@mail.gmail.com --- src/backend/storage/aio/method_io_uring.c | 89 +++++++++++++++++------ 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/src/backend/storage/aio/method_io_uring.c b/src/backend/storage/aio/method_io_uring.c index 39984df31b4..b42d28647b8 100644 --- a/src/backend/storage/aio/method_io_uring.c +++ b/src/backend/storage/aio/method_io_uring.c @@ -409,7 +409,6 @@ static int pgaio_uring_submit(uint16 num_staged_ios, PgAioHandle **staged_ios) { struct io_uring *uring_instance = &pgaio_my_uring_context->io_uring_ring; - int in_flight_before = dclist_count(&pgaio_my_backend->in_flight_ios); Assert(num_staged_ios <= PGAIO_SUBMIT_BATCH_SIZE); @@ -425,27 +424,6 @@ pgaio_uring_submit(uint16 num_staged_ios, PgAioHandle **staged_ios) pgaio_io_prepare_submit(ioh); pgaio_uring_sq_from_io(ioh, sqe); - - /* - * io_uring executes IO in process context if possible. That's - * generally good, as it reduces context switching. When performing a - * lot of buffered IO that means that copying between page cache and - * userspace memory happens in the foreground, as it can't be - * offloaded to DMA hardware as is possible when using direct IO. When - * executing a lot of buffered IO this causes io_uring to be slower - * than worker mode, as worker mode parallelizes the copying. io_uring - * can be told to offload work to worker threads instead. - * - * If an IO is buffered IO and we already have IOs in flight or - * multiple IOs are being submitted, we thus tell io_uring to execute - * the IO in the background. We don't do so for the first few IOs - * being submitted as executing in this process' context has lower - * latency. - */ - if (in_flight_before > 4 && (ioh->flags & PGAIO_HF_BUFFERED)) - io_uring_sqe_set_flags(sqe, IOSQE_ASYNC); - - in_flight_before++; } while (true) @@ -701,10 +679,63 @@ pgaio_uring_check_one(PgAioHandle *ioh, uint64 ref_generation) LWLockRelease(&owner_context->completion_lock); } +/* + * io_uring executes IO in process context if possible. That's generally good, + * as it reduces context switching. When performing a lot of buffered IO that + * means that copying between page cache and userspace memory happens in the + * foreground, as it can't be offloaded to DMA hardware as is possible when + * using direct IO. When executing a lot of buffered IO this causes io_uring + * to be slower than worker mode, as worker mode parallelizes the + * copying. io_uring can be told to offload work to worker threads instead. + * + * If the IOs are small, we only benefit from forcing things into the + * background if there is a lot of IO, as otherwise the overhead from context + * switching is higher than the gain. + * + * If IOs are large, there is benefit from asynchronous processing at lower + * queue depths, as IO latency is less of a crucial factor and parallelizing + * memory copies is more important. In addition, it is important to trigger + * asynchronous processing even at low queue depth, as with foreground + * processing we might never actually reach deep enough IO depths to trigger + * asynchronous processing, which in turn would deprive readahead control + * logic of information about whether a deeper look-ahead distance would be + * advantageous. + * + * We have done some basic benchmarking to validate the thresholds used, but + * it's quite plausible that there are better values. + */ +static bool +pgaio_uring_should_use_async(PgAioHandle *ioh, size_t io_size) +{ + /* + * With DIO there's no benefit from forcing asynchronous processing, as + * io_uring will never execute direct IO synchronously. + */ + if (!(ioh->flags & PGAIO_HF_BUFFERED)) + return false; + + /* + * Once the IO queue depth is not that shallow anymore, the overhead of + * dispatching to the background is a less significant factor. + */ + if (dclist_count(&pgaio_my_backend->in_flight_ios) > 4) + return true; + + /* + * If the IO is larger, the gains from parallelizing the memory copy are + * larger and typically the impact of the latency is smaller. + */ + if (io_size >= (BLCKSZ * 4)) + return true; + + return false; +} + static void pgaio_uring_sq_from_io(PgAioHandle *ioh, struct io_uring_sqe *sqe) { struct iovec *iov; + size_t io_size = 0; switch ((PgAioOp) ioh->op) { @@ -717,6 +748,8 @@ pgaio_uring_sq_from_io(PgAioHandle *ioh, struct io_uring_sqe *sqe) iov->iov_base, iov->iov_len, ioh->op_data.read.offset); + + io_size = iov->iov_len; } else { @@ -726,7 +759,13 @@ pgaio_uring_sq_from_io(PgAioHandle *ioh, struct io_uring_sqe *sqe) ioh->op_data.read.iov_length, ioh->op_data.read.offset); + for (int i = 0; i < ioh->op_data.read.iov_length; i++, iov++) + io_size += iov->iov_len; } + + if (pgaio_uring_should_use_async(ioh, io_size)) + io_uring_sqe_set_flags(sqe, IOSQE_ASYNC); + break; case PGAIO_OP_WRITEV: @@ -747,6 +786,12 @@ pgaio_uring_sq_from_io(PgAioHandle *ioh, struct io_uring_sqe *sqe) ioh->op_data.write.iov_length, ioh->op_data.write.offset); } + + /* + * For now don't trigger use of IOSQE_ASYNC for writes, it's not + * clear there is a performance benefit in doing so. + */ + break; case PGAIO_OP_INVALID: -- 2.53.0.1.gb2826b52eb