diff options
| -rw-r--r-- | fs/io-wq.c | 12 | ||||
| -rw-r--r-- | fs/io-wq.h | 2 | ||||
| -rw-r--r-- | fs/io_uring.c | 37 |
3 files changed, 34 insertions, 17 deletions
diff --git a/fs/io-wq.c b/fs/io-wq.c index c36bbcd823ce..800b299f9772 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -16,7 +16,6 @@ #include <linux/kthread.h> #include <linux/rculist_nulls.h> #include <linux/fs_struct.h> -#include <linux/task_work.h> #include <linux/blk-cgroup.h> #include <linux/audit.h> #include <linux/cpu.h> @@ -775,9 +774,6 @@ static int io_wq_manager(void *data) complete(&wq->done); while (!kthread_should_stop()) { - if (current->task_works) - task_work_run(); - for_each_node(node) { struct io_wqe *wqe = wq->wqes[node]; bool fork_worker[2] = { false, false }; @@ -800,9 +796,6 @@ static int io_wq_manager(void *data) schedule_timeout(HZ); } - if (current->task_works) - task_work_run(); - out: if (refcount_dec_and_test(&wq->refs)) { complete(&wq->done); @@ -1160,11 +1153,6 @@ void io_wq_destroy(struct io_wq *wq) __io_wq_destroy(wq); } -struct task_struct *io_wq_get_task(struct io_wq *wq) -{ - return wq->manager; -} - static bool io_wq_worker_affinity(struct io_worker *worker, void *data) { struct task_struct *task = worker->task; diff --git a/fs/io-wq.h b/fs/io-wq.h index 096f1021018e..a1610702f222 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -124,8 +124,6 @@ typedef bool (work_cancel_fn)(struct io_wq_work *, void *); enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, void *data, bool cancel_all); -struct task_struct *io_wq_get_task(struct io_wq *wq); - #if defined(CONFIG_IO_WQ) extern void io_wq_worker_sleeping(struct task_struct *); extern void io_wq_worker_running(struct task_struct *); diff --git a/fs/io_uring.c b/fs/io_uring.c index ace7494e7404..e3eb37304e24 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -456,6 +456,9 @@ struct io_ring_ctx { struct io_restriction restrictions; + /* exit task_work */ + struct callback_head *exit_task_work; + /* Keep this last, we don't need it for the fast path */ struct work_struct exit_work; }; @@ -2328,11 +2331,14 @@ static int io_req_task_work_add(struct io_kiocb *req) static void io_req_task_work_add_fallback(struct io_kiocb *req, task_work_func_t cb) { - struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq); + struct io_ring_ctx *ctx = req->ctx; + struct callback_head *head; init_task_work(&req->task_work, cb); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); + do { + head = READ_ONCE(ctx->exit_task_work); + req->task_work.next = head; + } while (cmpxchg(&ctx->exit_task_work, head, &req->task_work) != head); } static void __io_req_task_cancel(struct io_kiocb *req, int error) @@ -8835,6 +8841,28 @@ static int io_remove_personalities(int id, void *p, void *data) return 0; } +static void io_run_ctx_fallback(struct io_ring_ctx *ctx) +{ + struct callback_head *work, *head, *next; + + do { + do { + head = NULL; + work = READ_ONCE(ctx->exit_task_work); + } while (cmpxchg(&ctx->exit_task_work, work, head) != work); + + if (!work) + break; + + do { + next = work->next; + work->func(work); + work = next; + cond_resched(); + } while (work); + } while (1); +} + static void io_ring_exit_work(struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, @@ -8848,6 +8876,7 @@ static void io_ring_exit_work(struct work_struct *work) */ do { io_uring_try_cancel_requests(ctx, NULL, NULL); + io_run_ctx_fallback(ctx); } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); io_ring_ctx_free(ctx); } @@ -9243,6 +9272,8 @@ static int io_uring_flush(struct file *file, void *data) io_req_caches_free(ctx, current); } + io_run_ctx_fallback(ctx); + if (!tctx) return 0; |