diff --git a/block/Kconfig b/block/Kconfig index 1f2469a0123ceb1f36103c0db9eb71d140556193..da71e56f8682c029cd7d4b796d19b63fc9fd59a9 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -200,6 +200,19 @@ config BLK_SED_OPAL Enabling this option enables users to setup/unlock/lock Locking ranges for SED devices using the Opal protocol. +config BLK_BIO_DISPATCH_ASYNC + bool "Dispatch bios asynchronously on specific cpus" + default n + depends on BLOCK=y + help + If there are multiple nodes, memory access across nodes is rather bad + compare to local node. And if some drivers are using internal spin + locks, io performance will be bad if bios are issued concurrently from + different nodes. This feature will dispatch bio asynchronously to the + specific CPUs to avoid across nodes memory access in driver, noted this + feature will require special care in the driver to work. If unsure, + say N here. + menu "Partition Types" source "block/partitions/Kconfig" diff --git a/block/blk-core.c b/block/blk-core.c index 08e7a875b2293aab71e59b57068580721278bf83..2ce4ad4619c19269c94e8cacb7e67a7a8d7214cd 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -35,7 +35,6 @@ #include <linux/blk-cgroup.h> #include <linux/debugfs.h> #include <linux/bpf.h> -#include <linux/arch_topology.h> #define CREATE_TRACE_POINTS #include <trace/events/block.h> @@ -86,6 +85,9 @@ struct kmem_cache *blk_requestq_cachep; */ static struct workqueue_struct *kblockd_workqueue; +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC +#include <linux/arch_topology.h> + #define BIO_DISPATCH_MAX_LOOP 16 /* the minimum of cpus that dispatch async can be enabled */ #define MIN_DISPATCH_ASYNC_CPUS 16 @@ -309,6 +311,23 @@ void queue_init_dispatch_async_cpus(struct request_queue *q, int node) cpumask_setall(dispatch_async_cpus); } EXPORT_SYMBOL_GPL(queue_init_dispatch_async_cpus); +#else +static int blk_alloc_queue_dispatch_async(struct request_queue *q) +{ + return 0; +} + +static blk_qc_t blk_queue_do_make_request(struct bio *bio) +{ + struct request_queue *q = bio->bi_disk->queue; + + return q->make_request_fn(q, bio); +} + +static void init_blk_queue_async_dispatch(void) +{ +} +#endif /** * blk_queue_flag_set - atomically set a queue flag diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index cbd7de22c463f240ae9674461045a3bde93483a8..5bdae7c8fa27b891f9d093619778dfc40b751525 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -696,6 +696,7 @@ static struct queue_sysfs_entry queue_wb_lat_entry = { .store = queue_wb_lat_store, }; +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC static ssize_t queue_dispatch_async_cpus_show(struct request_queue *q, char *page) { @@ -731,6 +732,7 @@ static struct queue_sysfs_entry queue_dispatch_async_entry = { .attr = {.name = "dispatch_async", .mode = 0444 }, .show = queue_show_dispatch_async, }; +#endif #ifdef CONFIG_BLK_DEV_THROTTLING_LOW static struct queue_sysfs_entry throtl_sample_time_entry = { @@ -774,8 +776,10 @@ static struct attribute *default_attrs[] = { &queue_dax_entry.attr, &queue_wb_lat_entry.attr, &queue_poll_delay_entry.attr, +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC &queue_dispatch_async_cpus_entry.attr, &queue_dispatch_async_entry.attr, +#endif #ifdef CONFIG_BLK_DEV_THROTTLING_LOW &throtl_sample_time_entry.attr, #endif diff --git a/block/blk.h b/block/blk.h index cb1c9057d7889f613c1971743bdfbde2dc27e8cb..985e20980aff2b319f84f7b698b845c787cb60a1 100644 --- a/block/blk.h +++ b/block/blk.h @@ -49,9 +49,11 @@ struct request_queue_wrapper { struct mutex mq_freeze_lock; int mq_freeze_depth; +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC /* used when QUEUE_FLAG_DISPATCH_ASYNC is set */ struct cpumask dispatch_async_cpus; int __percpu *last_dispatch_cpu; +#endif }; #define queue_to_wrapper(q) \ @@ -464,6 +466,12 @@ extern int blk_iolatency_init(struct request_queue *q); static inline int blk_iolatency_init(struct request_queue *q) { return 0; } #endif +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC extern void blk_free_queue_dispatch_async(struct request_queue *q); +#else +static inline void blk_free_queue_dispatch_async(struct request_queue *q) +{ +} +#endif #endif /* BLK_INTERNAL_H */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 996e005750426123ebc69c870e80d46c64f9158c..41d235ee579ae91bc55f0504ce243b36cf54f5cb 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -785,7 +785,14 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q); extern void blk_set_pm_only(struct request_queue *q); extern void blk_clear_pm_only(struct request_queue *q); +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC extern void queue_init_dispatch_async_cpus(struct request_queue *q, int node); +#else +static inline void queue_init_dispatch_async_cpus(struct request_queue *q, + int node) +{ +} +#endif static inline int queue_in_flight(struct request_queue *q) {