block: add queue flag for disabling SG merging

If devices are not SG starved, we waste a lot of time potentially
collapsing SG segments. Enough that 1.5% of the CPU time goes
to this, at only 400K IOPS. Add a queue flag, QUEUE_FLAG_NO_SG_MERGE,
which just returns the number of vectors in a bio instead of looping
over all segments and checking for collapsible ones.

Add a BLK_MQ_F_SG_MERGE flag so that drivers can opt-in on the sg
merging, if they so desire.

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/block/blk-merge.c b/block/blk-merge.c
index 6c583f9..b3bf0df 100644 (file)
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -13,7 +13,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
                                             struct bio *bio)
 {
        struct bio_vec bv, bvprv = { NULL };
-       int cluster, high, highprv = 1;
+       int cluster, high, highprv = 1, no_sg_merge;
        unsigned int seg_size, nr_phys_segs;
        struct bio *fbio, *bbio;
        struct bvec_iter iter;
@@ -35,12 +35,21 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
        cluster = blk_queue_cluster(q);
        seg_size = 0;
        nr_phys_segs = 0;
+       no_sg_merge = test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
+       high = 0;
        for_each_bio(bio) {
                bio_for_each_segment(bv, bio, iter) {
+                       /*
+                        * If SG merging is disabled, each bio vector is
+                        * a segment
+                        */
+                       if (no_sg_merge)
+                               goto new_segment;
+
                        /*
                         * the trick here is making sure that a high page is
-                        * never considered part of another segment, since that
-                        * might change with the bounce page.
+                        * never considered part of another segment, since
+                        * that might change with the bounce page.
                         */
                        high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q);
                        if (!high && !highprv && cluster) {
@@ -84,11 +93,16 @@ void blk_recalc_rq_segments(struct request *rq)
 
 void blk_recount_segments(struct request_queue *q, struct bio *bio)
 {
-       struct bio *nxt = bio->bi_next;
+       if (test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags))
+               bio->bi_phys_segments = bio->bi_vcnt;
+       else {
+               struct bio *nxt = bio->bi_next;
+
+               bio->bi_next = NULL;
+               bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
+               bio->bi_next = nxt;
+       }
 
-       bio->bi_next = NULL;
-       bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
-       bio->bi_next = nxt;
        bio->bi_flags |= (1 << BIO_SEG_VALID);
 }
 EXPORT_SYMBOL(blk_recount_segments);

diff --cc block/blk-mq.c
Simple merge

diff --cc include/linux/blk-mq.h
Simple merge

diff --cc include/linux/blkdev.h
Simple merge