mirror of
https://github.com/torvalds/linux.git
synced 2026-04-18 06:44:00 -04:00
23acda7c22
Pull io_uring updates from Jens Axboe:
- Add a callback driven main loop for io_uring, and BPF struct_ops
on top to allow implementing custom event loop logic
- Decouple IOPOLL from being a ring-wide all-or-nothing setting,
allowing IOPOLL use cases to also issue certain white listed
non-polled opcodes
- Timeout improvements. Migrate internal timeout storage from
timespec64 to ktime_t for simpler arithmetic and avoid copying of
timespec data
- Zero-copy receive (zcrx) updates:
- Add a device-less mode (ZCRX_REG_NODEV) for testing and
experimentation where data flows through the copy fallback path
- Fix two-step unregistration regression, DMA length calculations,
xarray mark usage, and a potential 32-bit overflow in id
shifting
- Refactoring toward multi-area support: dedicated refill queue
struct, consolidated DMA syncing, netmem array refilling format,
and guard-based locking
- Zero-copy transmit (zctx) cleanup:
- Unify io_send_zc() and io_sendmsg_zc() into a single function
- Add vectorized registered buffer send for IORING_OP_SEND_ZC
- Add separate notification user_data via sqe->addr3 so
notification and completion CQEs can be distinguished without
extra reference counting
- Switch struct io_ring_ctx internal bitfields to explicit flag bits
with atomic-safe accessors, and annotate the known harmless races on
those flags
- Various optimizations caching ctx and other request fields in local
variables to avoid repeated loads, and cleanups for tctx setup, ring
fd registration, and read path early returns
* tag 'for-7.1/io_uring-20260411' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: (58 commits)
io_uring: unify getting ctx from passed in file descriptor
io_uring/register: don't get a reference to the registered ring fd
io_uring/tctx: clean up __io_uring_add_tctx_node() error handling
io_uring/tctx: have io_uring_alloc_task_context() return tctx
io_uring/timeout: use 'ctx' consistently
io_uring/rw: clean up __io_read() obsolete comment and early returns
io_uring/zcrx: use correct mmap off constants
io_uring/zcrx: use dma_len for chunk size calculation
io_uring/zcrx: don't clear not allocated niovs
io_uring/zcrx: don't use mark0 for allocating xarray
io_uring: cast id to u64 before shifting in io_allocate_rbuf_ring()
io_uring/zcrx: reject REG_NODEV with large rx_buf_size
io_uring/cancel: validate opcode for IORING_ASYNC_CANCEL_OP
io_uring/rsrc: use io_cache_free() to free node
io_uring/zcrx: rename zcrx [un]register functions
io_uring/zcrx: check ctrl op payload struct sizes
io_uring/zcrx: cache fallback availability in zcrx ctx
io_uring/zcrx: warn on a repeated area append
io_uring/zcrx: consolidate dma syncing
io_uring/zcrx: netmem array as refiling format
...
54 lines
1.4 KiB
C
54 lines
1.4 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#ifndef IOU_WAIT_H
|
|
#define IOU_WAIT_H
|
|
|
|
#include <linux/io_uring_types.h>
|
|
|
|
/*
|
|
* No waiters. It's larger than any valid value of the tw counter
|
|
* so that tests against ->cq_wait_nr would fail and skip wake_up().
|
|
*/
|
|
#define IO_CQ_WAKE_INIT (-1U)
|
|
/* Forced wake up if there is a waiter regardless of ->cq_wait_nr */
|
|
#define IO_CQ_WAKE_FORCE (IO_CQ_WAKE_INIT >> 1)
|
|
|
|
struct ext_arg {
|
|
size_t argsz;
|
|
struct timespec64 ts;
|
|
const sigset_t __user *sig;
|
|
ktime_t min_time;
|
|
bool ts_set;
|
|
bool iowait;
|
|
};
|
|
|
|
int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
|
|
struct ext_arg *ext_arg);
|
|
int io_run_task_work_sig(struct io_ring_ctx *ctx);
|
|
void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx);
|
|
void io_cqring_overflow_flush_locked(struct io_ring_ctx *ctx);
|
|
|
|
static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
|
|
{
|
|
struct io_rings *rings = io_get_rings(ctx);
|
|
return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
|
|
}
|
|
|
|
static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)
|
|
{
|
|
struct io_rings *rings = io_get_rings(ctx);
|
|
|
|
return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);
|
|
}
|
|
|
|
/*
|
|
* Reads the tail/head of the CQ ring while providing an acquire ordering,
|
|
* see comment at top of io_uring.c.
|
|
*/
|
|
static inline unsigned io_cqring_events(struct io_ring_ctx *ctx)
|
|
{
|
|
smp_rmb();
|
|
return __io_cqring_events(ctx);
|
|
}
|
|
|
|
#endif
|