drivers/md/raid10.c

   1 /*
   2  * raid10.c : Multiple Devices driver for Linux
   3  *
   4  * Copyright (C) 2000-2004 Neil Brown
   5  *
   6  * RAID-10 support for md.
   7  *
   8  * Base on code in raid1.c.  See raid1.c for further copyright information.
   9  *
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2, or (at your option)
  14  * any later version.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * (for example /usr/src/linux/COPYING); if not, write to the Free
  18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19  */
  20
  21 #include <linux/slab.h>
  22 #include <linux/delay.h>
  23 #include <linux/blkdev.h>
  24 #include <linux/seq_file.h>
  25 #include <linux/ratelimit.h>
  26 #include "md.h"
  27 #include "raid10.h"
  28 #include "raid0.h"
  29 #include "bitmap.h"
  30
  31 /*
  32  * RAID10 provides a combination of RAID0 and RAID1 functionality.
  33  * The layout of data is defined by
  34  *    chunk_size
  35  *    raid_disks
  36  *    near_copies (stored in low byte of layout)
  37  *    far_copies (stored in second byte of layout)
  38  *    far_offset (stored in bit 16 of layout )
  39  *
  40  * The data to be stored is divided into chunks using chunksize.
  41  * Each device is divided into far_copies sections.
  42  * In each section, chunks are laid out in a style similar to raid0, but
  43  * near_copies copies of each chunk is stored (each on a different drive).
  44  * The starting device for each section is offset near_copies from the starting
  45  * device of the previous section.
  46  * Thus they are (near_copies*far_copies) of each chunk, and each is on a different
  47  * drive.
  48  * near_copies and far_copies must be at least one, and their product is at most
  49  * raid_disks.
  50  *
  51  * If far_offset is true, then the far_copies are handled a bit differently.
  52  * The copies are still in different stripes, but instead of be very far apart
  53  * on disk, there are adjacent stripes.
  54  */
  55
  56 /*
  57  * Number of guaranteed r10bios in case of extreme VM load:
  58  */
  59 #define NR_RAID10_BIOS 256
  60
  61 static void allow_barrier(conf_t *conf);
  62 static void lower_barrier(conf_t *conf);
  63
  64 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
  65 {
  66         conf_t *conf = data;
  67         int size = offsetof(struct r10bio_s, devs[conf->copies]);
  68
  69         /* allocate a r10bio with room for raid_disks entries in the bios array */
  70         return kzalloc(size, gfp_flags);
  71 }
  72
  73 static void r10bio_pool_free(void *r10_bio, void *data)
  74 {
  75         kfree(r10_bio);
  76 }
  77
  78 /* Maximum size of each resync request */
  79 #define RESYNC_BLOCK_SIZE (64*1024)
  80 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
  81 /* amount of memory to reserve for resync requests */
  82 #define RESYNC_WINDOW (1024*1024)
  83 /* maximum number of concurrent requests, memory permitting */
  84 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
  85
  86 /*
  87  * When performing a resync, we need to read and compare, so
  88  * we need as many pages are there are copies.
  89  * When performing a recovery, we need 2 bios, one for read,
  90  * one for write (we recover only one drive per r10buf)
  91  *
  92  */
  93 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
  94 {
  95         conf_t *conf = data;
  96         struct page *page;
  97         r10bio_t *r10_bio;
  98         struct bio *bio;
  99         int i, j;
 100         int nalloc;
 101
 102         r10_bio = r10bio_pool_alloc(gfp_flags, conf);
 103         if (!r10_bio)
 104                 return NULL;
 105
 106         if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery))
 107                 nalloc = conf->copies; /* resync */
 108         else
 109                 nalloc = 2; /* recovery */
 110
 111         /*
 112          * Allocate bios.
 113          */
 114         for (j = nalloc ; j-- ; ) {
 115                 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
 116                 if (!bio)
 117                         goto out_free_bio;
 118                 r10_bio->devs[j].bio = bio;
 119         }
 120         /*
 121          * Allocate RESYNC_PAGES data pages and attach them
 122          * where needed.
 123          */
 124         for (j = 0 ; j < nalloc; j++) {
 125                 bio = r10_bio->devs[j].bio;
 126                 for (i = 0; i < RESYNC_PAGES; i++) {
 127                         if (j == 1 && !test_bit(MD_RECOVERY_SYNC,
 128                                                 &conf->mddev->recovery)) {
 129                                 /* we can share bv_page's during recovery */
 130                                 struct bio *rbio = r10_bio->devs[0].bio;
 131                                 page = rbio->bi_io_vec[i].bv_page;
 132                                 get_page(page);
 133                         } else
 134                                 page = alloc_page(gfp_flags);
 135                         if (unlikely(!page))
 136                                 goto out_free_pages;
 137
 138                         bio->bi_io_vec[i].bv_page = page;
 139                 }
 140         }
 141
 142         return r10_bio;
 143
 144 out_free_pages:
 145         for ( ; i > 0 ; i--)
 146                 safe_put_page(bio->bi_io_vec[i-1].bv_page);
 147         while (j--)
 148                 for (i = 0; i < RESYNC_PAGES ; i++)
 149                         safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
 150         j = -1;
 151 out_free_bio:
 152         while ( ++j < nalloc )
 153                 bio_put(r10_bio->devs[j].bio);
 154         r10bio_pool_free(r10_bio, conf);
 155         return NULL;
 156 }
 157
 158 static void r10buf_pool_free(void *__r10_bio, void *data)
 159 {
 160         int i;
 161         conf_t *conf = data;
 162         r10bio_t *r10bio = __r10_bio;
 163         int j;
 164
 165         for (j=0; j < conf->copies; j++) {
 166                 struct bio *bio = r10bio->devs[j].bio;
 167                 if (bio) {
 168                         for (i = 0; i < RESYNC_PAGES; i++) {
 169                                 safe_put_page(bio->bi_io_vec[i].bv_page);
 170                                 bio->bi_io_vec[i].bv_page = NULL;
 171                         }
 172                         bio_put(bio);
 173                 }
 174         }
 175         r10bio_pool_free(r10bio, conf);
 176 }
 177
 178 static void put_all_bios(conf_t *conf, r10bio_t *r10_bio)
 179 {
 180         int i;
 181
 182         for (i = 0; i < conf->copies; i++) {
 183                 struct bio **bio = & r10_bio->devs[i].bio;
 184                 if (!BIO_SPECIAL(*bio))
 185                         bio_put(*bio);
 186                 *bio = NULL;
 187         }
 188 }
 189
 190 static void free_r10bio(r10bio_t *r10_bio)
 191 {
 192         conf_t *conf = r10_bio->mddev->private;
 193
 194         put_all_bios(conf, r10_bio);
 195         mempool_free(r10_bio, conf->r10bio_pool);
 196 }
 197
 198 static void put_buf(r10bio_t *r10_bio)
 199 {
 200         conf_t *conf = r10_bio->mddev->private;
 201
 202         mempool_free(r10_bio, conf->r10buf_pool);
 203
 204         lower_barrier(conf);
 205 }
 206
 207 static void reschedule_retry(r10bio_t *r10_bio)
 208 {
 209         unsigned long flags;
 210         mddev_t *mddev = r10_bio->mddev;
 211         conf_t *conf = mddev->private;
 212
 213         spin_lock_irqsave(&conf->device_lock, flags);
 214         list_add(&r10_bio->retry_list, &conf->retry_list);
 215         conf->nr_queued ++;
 216         spin_unlock_irqrestore(&conf->device_lock, flags);
 217
 218         /* wake up frozen array... */
 219         wake_up(&conf->wait_barrier);
 220
 221         md_wakeup_thread(mddev->thread);
 222 }
 223
 224 /*
 225  * raid_end_bio_io() is called when we have finished servicing a mirrored
 226  * operation and are ready to return a success/failure code to the buffer
 227  * cache layer.
 228  */
 229 static void raid_end_bio_io(r10bio_t *r10_bio)
 230 {
 231         struct bio *bio = r10_bio->master_bio;
 232         int done;
 233         conf_t *conf = r10_bio->mddev->private;
 234
 235         if (bio->bi_phys_segments) {
 236                 unsigned long flags;
 237                 spin_lock_irqsave(&conf->device_lock, flags);
 238                 bio->bi_phys_segments--;
 239                 done = (bio->bi_phys_segments == 0);
 240                 spin_unlock_irqrestore(&conf->device_lock, flags);
 241         } else
 242                 done = 1;
 243         if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
 244                 clear_bit(BIO_UPTODATE, &bio->bi_flags);
 245         if (done) {
 246                 bio_endio(bio, 0);
 247                 /*
 248                  * Wake up any possible resync thread that waits for the device
 249                  * to go idle.
 250                  */
 251                 allow_barrier(conf);
 252         }
 253         free_r10bio(r10_bio);
 254 }
 255
 256 /*
 257  * Update disk head position estimator based on IRQ completion info.
 258  */
 259 static inline void update_head_pos(int slot, r10bio_t *r10_bio)
 260 {
 261         conf_t *conf = r10_bio->mddev->private;
 262
 263         conf->mirrors[r10_bio->devs[slot].devnum].head_position =
 264                 r10_bio->devs[slot].addr + (r10_bio->sectors);
 265 }
 266
 267 /*
 268  * Find the disk number which triggered given bio
 269  */
 270 static int find_bio_disk(conf_t *conf, r10bio_t *r10_bio,
 271                          struct bio *bio, int *slotp)
 272 {
 273         int slot;
 274
 275         for (slot = 0; slot < conf->copies; slot++)
 276                 if (r10_bio->devs[slot].bio == bio)
 277                         break;
 278
 279         BUG_ON(slot == conf->copies);
 280         update_head_pos(slot, r10_bio);
 281
 282         if (slotp)
 283                 *slotp = slot;
 284         return r10_bio->devs[slot].devnum;
 285 }
 286
 287 static void raid10_end_read_request(struct bio *bio, int error)
 288 {
 289         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 290         r10bio_t *r10_bio = bio->bi_private;
 291         int slot, dev;
 292         conf_t *conf = r10_bio->mddev->private;
 293
 294
 295         slot = r10_bio->read_slot;
 296         dev = r10_bio->devs[slot].devnum;
 297         /*
 298          * this branch is our 'one mirror IO has finished' event handler:
 299          */
 300         update_head_pos(slot, r10_bio);
 301
 302         if (uptodate) {
 303                 /*
 304                  * Set R10BIO_Uptodate in our master bio, so that
 305                  * we will return a good error code to the higher
 306                  * levels even if IO on some other mirrored buffer fails.
 307                  *
 308                  * The 'master' represents the composite IO operation to
 309                  * user-side. So if something waits for IO, then it will
 310                  * wait for the 'master' bio.
 311                  */
 312                 set_bit(R10BIO_Uptodate, &r10_bio->state);
 313                 raid_end_bio_io(r10_bio);
 314                 rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
 315         } else {
 316                 /*
 317                  * oops, read error - keep the refcount on the rdev
 318                  */
 319                 char b[BDEVNAME_SIZE];
 320                 printk_ratelimited(KERN_ERR
 321                                    "md/raid10:%s: %s: rescheduling sector %llu\n",
 322                                    mdname(conf->mddev),
 323                                    bdevname(conf->mirrors[dev].rdev->bdev, b),
 324                                    (unsigned long long)r10_bio->sector);
 325                 set_bit(R10BIO_ReadError, &r10_bio->state);
 326                 reschedule_retry(r10_bio);
 327         }
 328 }
 329
 330 static void close_write(r10bio_t *r10_bio)
 331 {
 332         /* clear the bitmap if all writes complete successfully */
 333         bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
 334                         r10_bio->sectors,
 335                         !test_bit(R10BIO_Degraded, &r10_bio->state),
 336                         0);
 337         md_write_end(r10_bio->mddev);
 338 }
 339
 340 static void raid10_end_write_request(struct bio *bio, int error)
 341 {
 342         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 343         r10bio_t *r10_bio = bio->bi_private;
 344         int dev;
 345         int dec_rdev = 1;
 346         conf_t *conf = r10_bio->mddev->private;
 347         int slot;
 348
 349         dev = find_bio_disk(conf, r10_bio, bio, &slot);
 350
 351         /*
 352          * this branch is our 'one mirror IO has finished' event handler:
 353          */
 354         if (!uptodate) {
 355                 set_bit(WriteErrorSeen, &conf->mirrors[dev].rdev->flags);
 356                 set_bit(R10BIO_WriteError, &r10_bio->state);
 357                 dec_rdev = 0;
 358         } else {
 359                 /*
 360                  * Set R10BIO_Uptodate in our master bio, so that
 361                  * we will return a good error code for to the higher
 362                  * levels even if IO on some other mirrored buffer fails.
 363                  *
 364                  * The 'master' represents the composite IO operation to
 365                  * user-side. So if something waits for IO, then it will
 366                  * wait for the 'master' bio.
 367                  */
 368                 sector_t first_bad;
 369                 int bad_sectors;
 370
 371                 set_bit(R10BIO_Uptodate, &r10_bio->state);
 372
 373                 /* Maybe we can clear some bad blocks. */
 374                 if (is_badblock(conf->mirrors[dev].rdev,
 375                                 r10_bio->devs[slot].addr,
 376                                 r10_bio->sectors,
 377                                 &first_bad, &bad_sectors)) {
 378                         bio_put(bio);
 379                         r10_bio->devs[slot].bio = IO_MADE_GOOD;
 380                         dec_rdev = 0;
 381                         set_bit(R10BIO_MadeGood, &r10_bio->state);
 382                 }
 383         }
 384
 385         /*
 386          *
 387          * Let's see if all mirrored write operations have finished
 388          * already.
 389          */
 390         if (atomic_dec_and_test(&r10_bio->remaining)) {
 391                 if (test_bit(R10BIO_WriteError, &r10_bio->state))
 392                         reschedule_retry(r10_bio);
 393                 else {
 394                         close_write(r10_bio);
 395                         if (test_bit(R10BIO_MadeGood, &r10_bio->state))
 396                                 reschedule_retry(r10_bio);
 397                         else
 398                                 raid_end_bio_io(r10_bio);
 399                 }
 400         }
 401         if (dec_rdev)
 402                 rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
 403 }
 404
 405
 406 /*
 407  * RAID10 layout manager
 408  * As well as the chunksize and raid_disks count, there are two
 409  * parameters: near_copies and far_copies.
 410  * near_copies * far_copies must be <= raid_disks.
 411  * Normally one of these will be 1.
 412  * If both are 1, we get raid0.
 413  * If near_copies == raid_disks, we get raid1.
 414  *
 415  * Chunks are laid out in raid0 style with near_copies copies of the
 416  * first chunk, followed by near_copies copies of the next chunk and
 417  * so on.
 418  * If far_copies > 1, then after 1/far_copies of the array has been assigned
 419  * as described above, we start again with a device offset of near_copies.
 420  * So we effectively have another copy of the whole array further down all
 421  * the drives, but with blocks on different drives.
 422  * With this layout, and block is never stored twice on the one device.
 423  *
 424  * raid10_find_phys finds the sector offset of a given virtual sector
 425  * on each device that it is on.
 426  *
 427  * raid10_find_virt does the reverse mapping, from a device and a
 428  * sector offset to a virtual address
 429  */
 430
 431 static void raid10_find_phys(conf_t *conf, r10bio_t *r10bio)
 432 {
 433         int n,f;
 434         sector_t sector;
 435         sector_t chunk;
 436         sector_t stripe;
 437         int dev;
 438
 439         int slot = 0;
 440
 441         /* now calculate first sector/dev */
 442         chunk = r10bio->sector >> conf->chunk_shift;
 443         sector = r10bio->sector & conf->chunk_mask;
 444
 445         chunk *= conf->near_copies;
 446         stripe = chunk;
 447         dev = sector_div(stripe, conf->raid_disks);
 448         if (conf->far_offset)
 449                 stripe *= conf->far_copies;
 450
 451         sector += stripe << conf->chunk_shift;
 452
 453         /* and calculate all the others */
 454         for (n=0; n < conf->near_copies; n++) {
 455                 int d = dev;
 456                 sector_t s = sector;
 457                 r10bio->devs[slot].addr = sector;
 458                 r10bio->devs[slot].devnum = d;
 459                 slot++;
 460
 461                 for (f = 1; f < conf->far_copies; f++) {
 462                         d += conf->near_copies;
 463                         if (d >= conf->raid_disks)
 464                                 d -= conf->raid_disks;
 465                         s += conf->stride;
 466                         r10bio->devs[slot].devnum = d;
 467                         r10bio->devs[slot].addr = s;
 468                         slot++;
 469                 }
 470                 dev++;
 471                 if (dev >= conf->raid_disks) {
 472                         dev = 0;
 473                         sector += (conf->chunk_mask + 1);
 474                 }
 475         }
 476         BUG_ON(slot != conf->copies);
 477 }
 478
 479 static sector_t raid10_find_virt(conf_t *conf, sector_t sector, int dev)
 480 {
 481         sector_t offset, chunk, vchunk;
 482
 483         offset = sector & conf->chunk_mask;
 484         if (conf->far_offset) {
 485                 int fc;
 486                 chunk = sector >> conf->chunk_shift;
 487                 fc = sector_div(chunk, conf->far_copies);
 488                 dev -= fc * conf->near_copies;
 489                 if (dev < 0)
 490                         dev += conf->raid_disks;
 491         } else {
 492                 while (sector >= conf->stride) {
 493                         sector -= conf->stride;
 494                         if (dev < conf->near_copies)
 495                                 dev += conf->raid_disks - conf->near_copies;
 496                         else
 497                                 dev -= conf->near_copies;
 498                 }
 499                 chunk = sector >> conf->chunk_shift;
 500         }
 501         vchunk = chunk * conf->raid_disks + dev;
 502         sector_div(vchunk, conf->near_copies);
 503         return (vchunk << conf->chunk_shift) + offset;
 504 }
 505
 506 /**
 507  *      raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
 508  *      @q: request queue
 509  *      @bvm: properties of new bio
 510  *      @biovec: the request that could be merged to it.
 511  *
 512  *      Return amount of bytes we can accept at this offset
 513  *      If near_copies == raid_disk, there are no striping issues,
 514  *      but in that case, the function isn't called at all.
 515  */
 516 static int raid10_mergeable_bvec(struct request_queue *q,
 517                                  struct bvec_merge_data *bvm,
 518                                  struct bio_vec *biovec)
 519 {
 520         mddev_t *mddev = q->queuedata;
 521         sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
 522         int max;
 523         unsigned int chunk_sectors = mddev->chunk_sectors;
 524         unsigned int bio_sectors = bvm->bi_size >> 9;
 525
 526         max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
 527         if (max < 0) max = 0; /* bio_add cannot handle a negative return */
 528         if (max <= biovec->bv_len && bio_sectors == 0)
 529                 return biovec->bv_len;
 530         else
 531                 return max;
 532 }
 533
 534 /*
 535  * This routine returns the disk from which the requested read should
 536  * be done. There is a per-array 'next expected sequential IO' sector
 537  * number - if this matches on the next IO then we use the last disk.
 538  * There is also a per-disk 'last know head position' sector that is
 539  * maintained from IRQ contexts, both the normal and the resync IO
 540  * completion handlers update this position correctly. If there is no
 541  * perfect sequential match then we pick the disk whose head is closest.
 542  *
 543  * If there are 2 mirrors in the same 2 devices, performance degrades
 544  * because position is mirror, not device based.
 545  *
 546  * The rdev for the device selected will have nr_pending incremented.
 547  */
 548
 549 /*
 550  * FIXME: possibly should rethink readbalancing and do it differently
 551  * depending on near_copies / far_copies geometry.
 552  */
 553 static int read_balance(conf_t *conf, r10bio_t *r10_bio, int *max_sectors)
 554 {
 555         const sector_t this_sector = r10_bio->sector;
 556         int disk, slot;
 557         int sectors = r10_bio->sectors;
 558         int best_good_sectors;
 559         sector_t new_distance, best_dist;
 560         mdk_rdev_t *rdev;
 561         int do_balance;
 562         int best_slot;
 563
 564         raid10_find_phys(conf, r10_bio);
 565         rcu_read_lock();
 566 retry:
 567         sectors = r10_bio->sectors;
 568         best_slot = -1;
 569         best_dist = MaxSector;
 570         best_good_sectors = 0;
 571         do_balance = 1;
 572         /*
 573          * Check if we can balance. We can balance on the whole
 574          * device if no resync is going on (recovery is ok), or below
 575          * the resync window. We take the first readable disk when
 576          * above the resync window.
 577          */
 578         if (conf->mddev->recovery_cp < MaxSector
 579             && (this_sector + sectors >= conf->next_resync))
 580                 do_balance = 0;
 581
 582         for (slot = 0; slot < conf->copies ; slot++) {
 583                 sector_t first_bad;
 584                 int bad_sectors;
 585                 sector_t dev_sector;
 586
 587                 if (r10_bio->devs[slot].bio == IO_BLOCKED)
 588                         continue;
 589                 disk = r10_bio->devs[slot].devnum;
 590                 rdev = rcu_dereference(conf->mirrors[disk].rdev);
 591                 if (rdev == NULL)
 592                         continue;
 593                 if (!test_bit(In_sync, &rdev->flags))
 594                         continue;
 595
 596                 dev_sector = r10_bio->devs[slot].addr;
 597                 if (is_badblock(rdev, dev_sector, sectors,
 598                                 &first_bad, &bad_sectors)) {
 599                         if (best_dist < MaxSector)
 600                                 /* Already have a better slot */
 601                                 continue;
 602                         if (first_bad <= dev_sector) {
 603                                 /* Cannot read here.  If this is the
 604                                  * 'primary' device, then we must not read
 605                                  * beyond 'bad_sectors' from another device.
 606                                  */
 607                                 bad_sectors -= (dev_sector - first_bad);
 608                                 if (!do_balance && sectors > bad_sectors)
 609                                         sectors = bad_sectors;
 610                                 if (best_good_sectors > sectors)
 611                                         best_good_sectors = sectors;
 612                         } else {
 613                                 sector_t good_sectors =
 614                                         first_bad - dev_sector;
 615                                 if (good_sectors > best_good_sectors) {
 616                                         best_good_sectors = good_sectors;
 617                                         best_slot = slot;
 618                                 }
 619                                 if (!do_balance)
 620                                         /* Must read from here */
 621                                         break;
 622                         }
 623                         continue;
 624                 } else
 625                         best_good_sectors = sectors;
 626
 627                 if (!do_balance)
 628                         break;
 629
 630                 /* This optimisation is debatable, and completely destroys
 631                  * sequential read speed for 'far copies' arrays.  So only
 632                  * keep it for 'near' arrays, and review those later.
 633                  */
 634                 if (conf->near_copies > 1 && !atomic_read(&rdev->nr_pending))
 635                         break;
 636
 637                 /* for far > 1 always use the lowest address */
 638                 if (conf->far_copies > 1)
 639                         new_distance = r10_bio->devs[slot].addr;
 640                 else
 641                         new_distance = abs(r10_bio->devs[slot].addr -
 642                                            conf->mirrors[disk].head_position);
 643                 if (new_distance < best_dist) {
 644                         best_dist = new_distance;
 645                         best_slot = slot;
 646                 }
 647         }
 648         if (slot == conf->copies)
 649                 slot = best_slot;
 650
 651         if (slot >= 0) {
 652                 disk = r10_bio->devs[slot].devnum;
 653                 rdev = rcu_dereference(conf->mirrors[disk].rdev);
 654                 if (!rdev)
 655                         goto retry;
 656                 atomic_inc(&rdev->nr_pending);
 657                 if (test_bit(Faulty, &rdev->flags)) {
 658                         /* Cannot risk returning a device that failed
 659                          * before we inc'ed nr_pending
 660                          */
 661                         rdev_dec_pending(rdev, conf->mddev);
 662                         goto retry;
 663                 }
 664                 r10_bio->read_slot = slot;
 665         } else
 666                 disk = -1;
 667         rcu_read_unlock();
 668         *max_sectors = best_good_sectors;
 669
 670         return disk;
 671 }
 672
 673 static int raid10_congested(void *data, int bits)
 674 {
 675         mddev_t *mddev = data;
 676         conf_t *conf = mddev->private;
 677         int i, ret = 0;
 678
 679         if (mddev_congested(mddev, bits))
 680                 return 1;
 681         rcu_read_lock();
 682         for (i = 0; i < conf->raid_disks && ret == 0; i++) {
 683                 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
 684                 if (rdev && !test_bit(Faulty, &rdev->flags)) {
 685                         struct request_queue *q = bdev_get_queue(rdev->bdev);
 686
 687                         ret |= bdi_congested(&q->backing_dev_info, bits);
 688                 }
 689         }
 690         rcu_read_unlock();
 691         return ret;
 692 }
 693
 694 static void flush_pending_writes(conf_t *conf)
 695 {
 696         /* Any writes that have been queued but are awaiting
 697          * bitmap updates get flushed here.
 698          */
 699         spin_lock_irq(&conf->device_lock);
 700
 701         if (conf->pending_bio_list.head) {
 702                 struct bio *bio;
 703                 bio = bio_list_get(&conf->pending_bio_list);
 704                 spin_unlock_irq(&conf->device_lock);
 705                 /* flush any pending bitmap writes to disk
 706                  * before proceeding w/ I/O */
 707                 bitmap_unplug(conf->mddev->bitmap);
 708
 709                 while (bio) { /* submit pending writes */
 710                         struct bio *next = bio->bi_next;
 711                         bio->bi_next = NULL;
 712                         generic_make_request(bio);
 713                         bio = next;
 714                 }
 715         } else
 716                 spin_unlock_irq(&conf->device_lock);
 717 }
 718
 719 /* Barriers....
 720  * Sometimes we need to suspend IO while we do something else,
 721  * either some resync/recovery, or reconfigure the array.
 722  * To do this we raise a 'barrier'.
 723  * The 'barrier' is a counter that can be raised multiple times
 724  * to count how many activities are happening which preclude
 725  * normal IO.
 726  * We can only raise the barrier if there is no pending IO.
 727  * i.e. if nr_pending == 0.
 728  * We choose only to raise the barrier if no-one is waiting for the
 729  * barrier to go down.  This means that as soon as an IO request
 730  * is ready, no other operations which require a barrier will start
 731  * until the IO request has had a chance.
 732  *
 733  * So: regular IO calls 'wait_barrier'.  When that returns there
 734  *    is no backgroup IO happening,  It must arrange to call
 735  *    allow_barrier when it has finished its IO.
 736  * backgroup IO calls must call raise_barrier.  Once that returns
 737  *    there is no normal IO happeing.  It must arrange to call
 738  *    lower_barrier when the particular background IO completes.
 739  */
 740
 741 static void raise_barrier(conf_t *conf, int force)
 742 {
 743         BUG_ON(force && !conf->barrier);
 744         spin_lock_irq(&conf->resync_lock);
 745
 746         /* Wait until no block IO is waiting (unless 'force') */
 747         wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
 748                             conf->resync_lock, );
 749
 750         /* block any new IO from starting */
 751         conf->barrier++;
 752
 753         /* Now wait for all pending IO to complete */
 754         wait_event_lock_irq(conf->wait_barrier,
 755                             !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
 756                             conf->resync_lock, );
 757
 758         spin_unlock_irq(&conf->resync_lock);
 759 }
 760
 761 static void lower_barrier(conf_t *conf)
 762 {
 763         unsigned long flags;
 764         spin_lock_irqsave(&conf->resync_lock, flags);
 765         conf->barrier--;
 766         spin_unlock_irqrestore(&conf->resync_lock, flags);
 767         wake_up(&conf->wait_barrier);
 768 }
 769
 770 static void wait_barrier(conf_t *conf)
 771 {
 772         spin_lock_irq(&conf->resync_lock);
 773         if (conf->barrier) {
 774                 conf->nr_waiting++;
 775                 wait_event_lock_irq(conf->wait_barrier, !conf->barrier,
 776                                     conf->resync_lock,
 777                                     );
 778                 conf->nr_waiting--;
 779         }
 780         conf->nr_pending++;
 781         spin_unlock_irq(&conf->resync_lock);
 782 }
 783
 784 static void allow_barrier(conf_t *conf)
 785 {
 786         unsigned long flags;
 787         spin_lock_irqsave(&conf->resync_lock, flags);
 788         conf->nr_pending--;
 789         spin_unlock_irqrestore(&conf->resync_lock, flags);
 790         wake_up(&conf->wait_barrier);
 791 }
 792
 793 static void freeze_array(conf_t *conf)
 794 {
 795         /* stop syncio and normal IO and wait for everything to
 796          * go quiet.
 797          * We increment barrier and nr_waiting, and then
 798          * wait until nr_pending match nr_queued+1
 799          * This is called in the context of one normal IO request
 800          * that has failed. Thus any sync request that might be pending
 801          * will be blocked by nr_pending, and we need to wait for
 802          * pending IO requests to complete or be queued for re-try.
 803          * Thus the number queued (nr_queued) plus this request (1)
 804          * must match the number of pending IOs (nr_pending) before
 805          * we continue.
 806          */
 807         spin_lock_irq(&conf->resync_lock);
 808         conf->barrier++;
 809         conf->nr_waiting++;
 810         wait_event_lock_irq(conf->wait_barrier,
 811                             conf->nr_pending == conf->nr_queued+1,
 812                             conf->resync_lock,
 813                             flush_pending_writes(conf));
 814
 815         spin_unlock_irq(&conf->resync_lock);
 816 }
 817
 818 static void unfreeze_array(conf_t *conf)
 819 {
 820         /* reverse the effect of the freeze */
 821         spin_lock_irq(&conf->resync_lock);
 822         conf->barrier--;
 823         conf->nr_waiting--;
 824         wake_up(&conf->wait_barrier);
 825         spin_unlock_irq(&conf->resync_lock);
 826 }
 827
 828 static int make_request(mddev_t *mddev, struct bio * bio)
 829 {
 830         conf_t *conf = mddev->private;
 831         mirror_info_t *mirror;
 832         r10bio_t *r10_bio;
 833         struct bio *read_bio;
 834         int i;
 835         int chunk_sects = conf->chunk_mask + 1;
 836         const int rw = bio_data_dir(bio);
 837         const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
 838         const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
 839         unsigned long flags;
 840         mdk_rdev_t *blocked_rdev;
 841         int plugged;
 842         int sectors_handled;
 843         int max_sectors;
 844
 845         if (unlikely(bio->bi_rw & REQ_FLUSH)) {
 846                 md_flush_request(mddev, bio);
 847                 return 0;
 848         }
 849
 850         /* If this request crosses a chunk boundary, we need to
 851          * split it.  This will only happen for 1 PAGE (or less) requests.
 852          */
 853         if (unlikely( (bio->bi_sector & conf->chunk_mask) + (bio->bi_size >> 9)
 854                       > chunk_sects &&
 855                     conf->near_copies < conf->raid_disks)) {
 856                 struct bio_pair *bp;
 857                 /* Sanity check -- queue functions should prevent this happening */
 858                 if (bio->bi_vcnt != 1 ||
 859                     bio->bi_idx != 0)
 860                         goto bad_map;
 861                 /* This is a one page bio that upper layers
 862                  * refuse to split for us, so we need to split it.
 863                  */
 864                 bp = bio_split(bio,
 865                                chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
 866
 867                 /* Each of these 'make_request' calls will call 'wait_barrier'.
 868                  * If the first succeeds but the second blocks due to the resync
 869                  * thread raising the barrier, we will deadlock because the
 870                  * IO to the underlying device will be queued in generic_make_request
 871                  * and will never complete, so will never reduce nr_pending.
 872                  * So increment nr_waiting here so no new raise_barriers will
 873                  * succeed, and so the second wait_barrier cannot block.
 874                  */
 875                 spin_lock_irq(&conf->resync_lock);
 876                 conf->nr_waiting++;
 877                 spin_unlock_irq(&conf->resync_lock);
 878
 879                 if (make_request(mddev, &bp->bio1))
 880                         generic_make_request(&bp->bio1);
 881                 if (make_request(mddev, &bp->bio2))
 882                         generic_make_request(&bp->bio2);
 883
 884                 spin_lock_irq(&conf->resync_lock);
 885                 conf->nr_waiting--;
 886                 wake_up(&conf->wait_barrier);
 887                 spin_unlock_irq(&conf->resync_lock);
 888
 889                 bio_pair_release(bp);
 890                 return 0;
 891         bad_map:
 892                 printk("md/raid10:%s: make_request bug: can't convert block across chunks"
 893                        " or bigger than %dk %llu %d\n", mdname(mddev), chunk_sects/2,
 894                        (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
 895
 896                 bio_io_error(bio);
 897                 return 0;
 898         }
 899
 900         md_write_start(mddev, bio);
 901
 902         /*
 903          * Register the new request and wait if the reconstruction
 904          * thread has put up a bar for new requests.
 905          * Continue immediately if no resync is active currently.
 906          */
 907         wait_barrier(conf);
 908
 909         r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
 910
 911         r10_bio->master_bio = bio;
 912         r10_bio->sectors = bio->bi_size >> 9;
 913
 914         r10_bio->mddev = mddev;
 915         r10_bio->sector = bio->bi_sector;
 916         r10_bio->state = 0;
 917
 918         /* We might need to issue multiple reads to different
 919          * devices if there are bad blocks around, so we keep
 920          * track of the number of reads in bio->bi_phys_segments.
 921          * If this is 0, there is only one r10_bio and no locking
 922          * will be needed when the request completes.  If it is
 923          * non-zero, then it is the number of not-completed requests.
 924          */
 925         bio->bi_phys_segments = 0;
 926         clear_bit(BIO_SEG_VALID, &bio->bi_flags);
 927
 928         if (rw == READ) {
 929                 /*
 930                  * read balancing logic:
 931                  */
 932                 int disk;
 933                 int slot;
 934
 935 read_again:
 936                 disk = read_balance(conf, r10_bio, &max_sectors);
 937                 slot = r10_bio->read_slot;
 938                 if (disk < 0) {
 939                         raid_end_bio_io(r10_bio);
 940                         return 0;
 941                 }
 942                 mirror = conf->mirrors + disk;
 943
 944                 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
 945                 md_trim_bio(read_bio, r10_bio->sector - bio->bi_sector,
 946                             max_sectors);
 947
 948                 r10_bio->devs[slot].bio = read_bio;
 949
 950                 read_bio->bi_sector = r10_bio->devs[slot].addr +
 951                         mirror->rdev->data_offset;
 952                 read_bio->bi_bdev = mirror->rdev->bdev;
 953                 read_bio->bi_end_io = raid10_end_read_request;
 954                 read_bio->bi_rw = READ | do_sync;
 955                 read_bio->bi_private = r10_bio;
 956
 957                 if (max_sectors < r10_bio->sectors) {
 958                         /* Could not read all from this device, so we will
 959                          * need another r10_bio.
 960                          */
 961                         sectors_handled = (r10_bio->sectors + max_sectors
 962                                            - bio->bi_sector);
 963                         r10_bio->sectors = max_sectors;
 964                         spin_lock_irq(&conf->device_lock);
 965                         if (bio->bi_phys_segments == 0)
 966                                 bio->bi_phys_segments = 2;
 967                         else
 968                                 bio->bi_phys_segments++;
 969                         spin_unlock(&conf->device_lock);
 970                         /* Cannot call generic_make_request directly
 971                          * as that will be queued in __generic_make_request
 972                          * and subsequent mempool_alloc might block
 973                          * waiting for it.  so hand bio over to raid10d.
 974                          */
 975                         reschedule_retry(r10_bio);
 976
 977                         r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
 978
 979                         r10_bio->master_bio = bio;
 980                         r10_bio->sectors = ((bio->bi_size >> 9)
 981                                             - sectors_handled);
 982                         r10_bio->state = 0;
 983                         r10_bio->mddev = mddev;
 984                         r10_bio->sector = bio->bi_sector + sectors_handled;
 985                         goto read_again;
 986                 } else
 987                         generic_make_request(read_bio);
 988                 return 0;
 989         }
 990
 991         /*
 992          * WRITE:
 993          */
 994         /* first select target devices under rcu_lock and
 995          * inc refcount on their rdev.  Record them by setting
 996          * bios[x] to bio
 997          * If there are known/acknowledged bad blocks on any device
 998          * on which we have seen a write error, we want to avoid
 999          * writing to those blocks.  This potentially requires several
1000          * writes to write around the bad blocks.  Each set of writes
1001          * gets its own r10_bio with a set of bios attached.  The number
1002          * of r10_bios is recored in bio->bi_phys_segments just as with
1003          * the read case.
1004          */
1005         plugged = mddev_check_plugged(mddev);
1006
1007         raid10_find_phys(conf, r10_bio);
1008 retry_write:
1009         blocked_rdev = NULL;
1010         rcu_read_lock();
1011         max_sectors = r10_bio->sectors;
1012
1013         for (i = 0;  i < conf->copies; i++) {
1014                 int d = r10_bio->devs[i].devnum;
1015                 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[d].rdev);
1016                 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1017                         atomic_inc(&rdev->nr_pending);
1018                         blocked_rdev = rdev;
1019                         break;
1020                 }
1021                 r10_bio->devs[i].bio = NULL;
1022                 if (!rdev || test_bit(Faulty, &rdev->flags)) {
1023                         set_bit(R10BIO_Degraded, &r10_bio->state);
1024                         continue;
1025                 }
1026                 if (test_bit(WriteErrorSeen, &rdev->flags)) {
1027                         sector_t first_bad;
1028                         sector_t dev_sector = r10_bio->devs[i].addr;
1029                         int bad_sectors;
1030                         int is_bad;
1031
1032                         is_bad = is_badblock(rdev, dev_sector,
1033                                              max_sectors,
1034                                              &first_bad, &bad_sectors);
1035                         if (is_bad < 0) {
1036                                 /* Mustn't write here until the bad block
1037                                  * is acknowledged
1038                                  */
1039                                 atomic_inc(&rdev->nr_pending);
1040                                 set_bit(BlockedBadBlocks, &rdev->flags);
1041                                 blocked_rdev = rdev;
1042                                 break;
1043                         }
1044                         if (is_bad && first_bad <= dev_sector) {
1045                                 /* Cannot write here at all */
1046                                 bad_sectors -= (dev_sector - first_bad);
1047                                 if (bad_sectors < max_sectors)
1048                                         /* Mustn't write more than bad_sectors
1049                                          * to other devices yet
1050                                          */
1051                                         max_sectors = bad_sectors;
1052                                 /* We don't set R10BIO_Degraded as that
1053                                  * only applies if the disk is missing,
1054                                  * so it might be re-added, and we want to
1055                                  * know to recover this chunk.
1056                                  * In this case the device is here, and the
1057                                  * fact that this chunk is not in-sync is
1058                                  * recorded in the bad block log.
1059                                  */
1060                                 continue;
1061                         }
1062                         if (is_bad) {
1063                                 int good_sectors = first_bad - dev_sector;
1064                                 if (good_sectors < max_sectors)
1065                                         max_sectors = good_sectors;
1066                         }
1067                 }
1068                 r10_bio->devs[i].bio = bio;
1069                 atomic_inc(&rdev->nr_pending);
1070         }
1071         rcu_read_unlock();
1072
1073         if (unlikely(blocked_rdev)) {
1074                 /* Have to wait for this device to get unblocked, then retry */
1075                 int j;
1076                 int d;
1077
1078                 for (j = 0; j < i; j++)
1079                         if (r10_bio->devs[j].bio) {
1080                                 d = r10_bio->devs[j].devnum;
1081                                 rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1082                         }
1083                 allow_barrier(conf);
1084                 md_wait_for_blocked_rdev(blocked_rdev, mddev);
1085                 wait_barrier(conf);
1086                 goto retry_write;
1087         }
1088
1089         if (max_sectors < r10_bio->sectors) {
1090                 /* We are splitting this into multiple parts, so
1091                  * we need to prepare for allocating another r10_bio.
1092                  */
1093                 r10_bio->sectors = max_sectors;
1094                 spin_lock_irq(&conf->device_lock);
1095                 if (bio->bi_phys_segments == 0)
1096                         bio->bi_phys_segments = 2;
1097                 else
1098                         bio->bi_phys_segments++;
1099                 spin_unlock_irq(&conf->device_lock);
1100         }
1101         sectors_handled = r10_bio->sector + max_sectors - bio->bi_sector;
1102
1103         atomic_set(&r10_bio->remaining, 1);
1104         bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1105
1106         for (i = 0; i < conf->copies; i++) {
1107                 struct bio *mbio;
1108                 int d = r10_bio->devs[i].devnum;
1109                 if (!r10_bio->devs[i].bio)
1110                         continue;
1111
1112                 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1113                 md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1114                             max_sectors);
1115                 r10_bio->devs[i].bio = mbio;
1116
1117                 mbio->bi_sector = (r10_bio->devs[i].addr+
1118                                    conf->mirrors[d].rdev->data_offset);
1119                 mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
1120                 mbio->bi_end_io = raid10_end_write_request;
1121                 mbio->bi_rw = WRITE | do_sync | do_fua;
1122                 mbio->bi_private = r10_bio;
1123
1124                 atomic_inc(&r10_bio->remaining);
1125                 spin_lock_irqsave(&conf->device_lock, flags);
1126                 bio_list_add(&conf->pending_bio_list, mbio);
1127                 spin_unlock_irqrestore(&conf->device_lock, flags);
1128         }
1129
1130         if (atomic_dec_and_test(&r10_bio->remaining)) {
1131                 /* This matches the end of raid10_end_write_request() */
1132                 bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
1133                                 r10_bio->sectors,
1134                                 !test_bit(R10BIO_Degraded, &r10_bio->state),
1135                                 0);
1136                 md_write_end(mddev);
1137                 raid_end_bio_io(r10_bio);
1138         }
1139
1140         /* In case raid10d snuck in to freeze_array */
1141         wake_up(&conf->wait_barrier);
1142
1143         if (sectors_handled < (bio->bi_size >> 9)) {
1144                 /* We need another r10_bio.  It has already been counted
1145                  * in bio->bi_phys_segments.
1146                  */
1147                 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1148
1149                 r10_bio->master_bio = bio;
1150                 r10_bio->sectors = (bio->bi_size >> 9) - sectors_handled;
1151
1152                 r10_bio->mddev = mddev;
1153                 r10_bio->sector = bio->bi_sector + sectors_handled;
1154                 r10_bio->state = 0;
1155                 goto retry_write;
1156         }
1157
1158         if (do_sync || !mddev->bitmap || !plugged)
1159                 md_wakeup_thread(mddev->thread);
1160         return 0;
1161 }
1162
1163 static void status(struct seq_file *seq, mddev_t *mddev)
1164 {
1165         conf_t *conf = mddev->private;
1166         int i;
1167
1168         if (conf->near_copies < conf->raid_disks)
1169                 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1170         if (conf->near_copies > 1)
1171                 seq_printf(seq, " %d near-copies", conf->near_copies);
1172         if (conf->far_copies > 1) {
1173                 if (conf->far_offset)
1174                         seq_printf(seq, " %d offset-copies", conf->far_copies);
1175                 else
1176                         seq_printf(seq, " %d far-copies", conf->far_copies);
1177         }
1178         seq_printf(seq, " [%d/%d] [", conf->raid_disks,
1179                                         conf->raid_disks - mddev->degraded);
1180         for (i = 0; i < conf->raid_disks; i++)
1181                 seq_printf(seq, "%s",
1182                               conf->mirrors[i].rdev &&
1183                               test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
1184         seq_printf(seq, "]");
1185 }
1186
1187 /* check if there are enough drives for
1188  * every block to appear on atleast one.
1189  * Don't consider the device numbered 'ignore'
1190  * as we might be about to remove it.
1191  */
1192 static int enough(conf_t *conf, int ignore)
1193 {
1194         int first = 0;
1195
1196         do {
1197                 int n = conf->copies;
1198                 int cnt = 0;
1199                 while (n--) {
1200                         if (conf->mirrors[first].rdev &&
1201                             first != ignore)
1202                                 cnt++;
1203                         first = (first+1) % conf->raid_disks;
1204                 }
1205                 if (cnt == 0)
1206                         return 0;
1207         } while (first != 0);
1208         return 1;
1209 }
1210
1211 static void error(mddev_t *mddev, mdk_rdev_t *rdev)
1212 {
1213         char b[BDEVNAME_SIZE];
1214         conf_t *conf = mddev->private;
1215
1216         /*
1217          * If it is not operational, then we have already marked it as dead
1218          * else if it is the last working disks, ignore the error, let the
1219          * next level up know.
1220          * else mark the drive as failed
1221          */
1222         if (test_bit(In_sync, &rdev->flags)
1223             && !enough(conf, rdev->raid_disk))
1224                 /*
1225                  * Don't fail the drive, just return an IO error.
1226                  */
1227                 return;
1228         if (test_and_clear_bit(In_sync, &rdev->flags)) {
1229                 unsigned long flags;
1230                 spin_lock_irqsave(&conf->device_lock, flags);
1231                 mddev->degraded++;
1232                 spin_unlock_irqrestore(&conf->device_lock, flags);
1233                 /*
1234                  * if recovery is running, make sure it aborts.
1235                  */
1236                 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1237         }
1238         set_bit(Blocked, &rdev->flags);
1239         set_bit(Faulty, &rdev->flags);
1240         set_bit(MD_CHANGE_DEVS, &mddev->flags);
1241         printk(KERN_ALERT
1242                "md/raid10:%s: Disk failure on %s, disabling device.\n"
1243                "md/raid10:%s: Operation continuing on %d devices.\n",
1244                mdname(mddev), bdevname(rdev->bdev, b),
1245                mdname(mddev), conf->raid_disks - mddev->degraded);
1246 }
1247
1248 static void print_conf(conf_t *conf)
1249 {
1250         int i;
1251         mirror_info_t *tmp;
1252
1253         printk(KERN_DEBUG "RAID10 conf printout:\n");
1254         if (!conf) {
1255                 printk(KERN_DEBUG "(!conf)\n");
1256                 return;
1257         }
1258         printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
1259                 conf->raid_disks);
1260
1261         for (i = 0; i < conf->raid_disks; i++) {
1262                 char b[BDEVNAME_SIZE];
1263                 tmp = conf->mirrors + i;
1264                 if (tmp->rdev)
1265                         printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1266                                 i, !test_bit(In_sync, &tmp->rdev->flags),
1267                                 !test_bit(Faulty, &tmp->rdev->flags),
1268                                 bdevname(tmp->rdev->bdev,b));
1269         }
1270 }
1271
1272 static void close_sync(conf_t *conf)
1273 {
1274         wait_barrier(conf);
1275         allow_barrier(conf);
1276
1277         mempool_destroy(conf->r10buf_pool);
1278         conf->r10buf_pool = NULL;
1279 }
1280
1281 static int raid10_spare_active(mddev_t *mddev)
1282 {
1283         int i;
1284         conf_t *conf = mddev->private;
1285         mirror_info_t *tmp;
1286         int count = 0;
1287         unsigned long flags;
1288
1289         /*
1290          * Find all non-in_sync disks within the RAID10 configuration
1291          * and mark them in_sync
1292          */
1293         for (i = 0; i < conf->raid_disks; i++) {
1294                 tmp = conf->mirrors + i;
1295                 if (tmp->rdev
1296                     && !test_bit(Faulty, &tmp->rdev->flags)
1297                     && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1298                         count++;
1299                         sysfs_notify_dirent(tmp->rdev->sysfs_state);
1300                 }
1301         }
1302         spin_lock_irqsave(&conf->device_lock, flags);
1303         mddev->degraded -= count;
1304         spin_unlock_irqrestore(&conf->device_lock, flags);
1305
1306         print_conf(conf);
1307         return count;
1308 }
1309
1310
1311 static int raid10_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
1312 {
1313         conf_t *conf = mddev->private;
1314         int err = -EEXIST;
1315         int mirror;
1316         int first = 0;
1317         int last = conf->raid_disks - 1;
1318
1319         if (mddev->recovery_cp < MaxSector)
1320                 /* only hot-add to in-sync arrays, as recovery is
1321                  * very different from resync
1322                  */
1323                 return -EBUSY;
1324         if (!enough(conf, -1))
1325                 return -EINVAL;
1326
1327         if (rdev->raid_disk >= 0)
1328                 first = last = rdev->raid_disk;
1329
1330         if (rdev->saved_raid_disk >= first &&
1331             conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1332                 mirror = rdev->saved_raid_disk;
1333         else
1334                 mirror = first;
1335         for ( ; mirror <= last ; mirror++) {
1336                 mirror_info_t *p = &conf->mirrors[mirror];
1337                 if (p->recovery_disabled == mddev->recovery_disabled)
1338                         continue;
1339                 if (!p->rdev)
1340                         continue;
1341
1342                 disk_stack_limits(mddev->gendisk, rdev->bdev,
1343                                   rdev->data_offset << 9);
1344                 /* as we don't honour merge_bvec_fn, we must
1345                  * never risk violating it, so limit
1346                  * ->max_segments to one lying with a single
1347                  * page, as a one page request is never in
1348                  * violation.
1349                  */
1350                 if (rdev->bdev->bd_disk->queue->merge_bvec_fn) {
1351                         blk_queue_max_segments(mddev->queue, 1);
1352                         blk_queue_segment_boundary(mddev->queue,
1353                                                    PAGE_CACHE_SIZE - 1);
1354                 }
1355
1356                 p->head_position = 0;
1357                 rdev->raid_disk = mirror;
1358                 err = 0;
1359                 if (rdev->saved_raid_disk != mirror)
1360                         conf->fullsync = 1;
1361                 rcu_assign_pointer(p->rdev, rdev);
1362                 break;
1363         }
1364
1365         md_integrity_add_rdev(rdev, mddev);
1366         print_conf(conf);
1367         return err;
1368 }
1369
1370 static int raid10_remove_disk(mddev_t *mddev, int number)
1371 {
1372         conf_t *conf = mddev->private;
1373         int err = 0;
1374         mdk_rdev_t *rdev;
1375         mirror_info_t *p = conf->mirrors+ number;
1376
1377         print_conf(conf);
1378         rdev = p->rdev;
1379         if (rdev) {
1380                 if (test_bit(In_sync, &rdev->flags) ||
1381                     atomic_read(&rdev->nr_pending)) {
1382                         err = -EBUSY;
1383                         goto abort;
1384                 }
1385                 /* Only remove faulty devices in recovery
1386                  * is not possible.
1387                  */
1388                 if (!test_bit(Faulty, &rdev->flags) &&
1389                     mddev->recovery_disabled != p->recovery_disabled &&
1390                     enough(conf, -1)) {
1391                         err = -EBUSY;
1392                         goto abort;
1393                 }
1394                 p->rdev = NULL;
1395                 synchronize_rcu();
1396                 if (atomic_read(&rdev->nr_pending)) {
1397                         /* lost the race, try later */
1398                         err = -EBUSY;
1399                         p->rdev = rdev;
1400                         goto abort;
1401                 }
1402                 err = md_integrity_register(mddev);
1403         }
1404 abort:
1405
1406         print_conf(conf);
1407         return err;
1408 }
1409
1410
1411 static void end_sync_read(struct bio *bio, int error)
1412 {
1413         r10bio_t *r10_bio = bio->bi_private;
1414         conf_t *conf = r10_bio->mddev->private;
1415         int d;
1416
1417         d = find_bio_disk(conf, r10_bio, bio, NULL);
1418
1419         if (test_bit(BIO_UPTODATE, &bio->bi_flags))
1420                 set_bit(R10BIO_Uptodate, &r10_bio->state);
1421         else
1422                 /* The write handler will notice the lack of
1423                  * R10BIO_Uptodate and record any errors etc
1424                  */
1425                 atomic_add(r10_bio->sectors,
1426                            &conf->mirrors[d].rdev->corrected_errors);
1427
1428         /* for reconstruct, we always reschedule after a read.
1429          * for resync, only after all reads
1430          */
1431         rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
1432         if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
1433             atomic_dec_and_test(&r10_bio->remaining)) {
1434                 /* we have read all the blocks,
1435                  * do the comparison in process context in raid10d
1436                  */
1437                 reschedule_retry(r10_bio);
1438         }
1439 }
1440
1441 static void end_sync_request(r10bio_t *r10_bio)
1442 {
1443         mddev_t *mddev = r10_bio->mddev;
1444
1445         while (atomic_dec_and_test(&r10_bio->remaining)) {
1446                 if (r10_bio->master_bio == NULL) {
1447                         /* the primary of several recovery bios */
1448                         sector_t s = r10_bio->sectors;
1449                         if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1450                             test_bit(R10BIO_WriteError, &r10_bio->state))
1451                                 reschedule_retry(r10_bio);
1452                         else
1453                                 put_buf(r10_bio);
1454                         md_done_sync(mddev, s, 1);
1455                         break;
1456                 } else {
1457                         r10bio_t *r10_bio2 = (r10bio_t *)r10_bio->master_bio;
1458                         if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1459                             test_bit(R10BIO_WriteError, &r10_bio->state))
1460                                 reschedule_retry(r10_bio);
1461                         else
1462                                 put_buf(r10_bio);
1463                         r10_bio = r10_bio2;
1464                 }
1465         }
1466 }
1467
1468 static void end_sync_write(struct bio *bio, int error)
1469 {
1470         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1471         r10bio_t *r10_bio = bio->bi_private;
1472         mddev_t *mddev = r10_bio->mddev;
1473         conf_t *conf = mddev->private;
1474         int d;
1475         sector_t first_bad;
1476         int bad_sectors;
1477         int slot;
1478
1479         d = find_bio_disk(conf, r10_bio, bio, &slot);
1480
1481         if (!uptodate) {
1482                 set_bit(WriteErrorSeen, &conf->mirrors[d].rdev->flags);
1483                 set_bit(R10BIO_WriteError, &r10_bio->state);
1484         } else if (is_badblock(conf->mirrors[d].rdev,
1485                              r10_bio->devs[slot].addr,
1486                              r10_bio->sectors,
1487                              &first_bad, &bad_sectors))
1488                 set_bit(R10BIO_MadeGood, &r10_bio->state);
1489
1490         rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1491
1492         end_sync_request(r10_bio);
1493 }
1494
1495 /*
1496  * Note: sync and recover and handled very differently for raid10
1497  * This code is for resync.
1498  * For resync, we read through virtual addresses and read all blocks.
1499  * If there is any error, we schedule a write.  The lowest numbered
1500  * drive is authoritative.
1501  * However requests come for physical address, so we need to map.
1502  * For every physical address there are raid_disks/copies virtual addresses,
1503  * which is always are least one, but is not necessarly an integer.
1504  * This means that a physical address can span multiple chunks, so we may
1505  * have to submit multiple io requests for a single sync request.
1506  */
1507 /*
1508  * We check if all blocks are in-sync and only write to blocks that
1509  * aren't in sync
1510  */
1511 static void sync_request_write(mddev_t *mddev, r10bio_t *r10_bio)
1512 {
1513         conf_t *conf = mddev->private;
1514         int i, first;
1515         struct bio *tbio, *fbio;
1516
1517         atomic_set(&r10_bio->remaining, 1);
1518
1519         /* find the first device with a block */
1520         for (i=0; i<conf->copies; i++)
1521                 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
1522                         break;
1523
1524         if (i == conf->copies)
1525                 goto done;
1526
1527         first = i;
1528         fbio = r10_bio->devs[i].bio;
1529
1530         /* now find blocks with errors */
1531         for (i=0 ; i < conf->copies ; i++) {
1532                 int  j, d;
1533                 int vcnt = r10_bio->sectors >> (PAGE_SHIFT-9);
1534
1535                 tbio = r10_bio->devs[i].bio;
1536
1537                 if (tbio->bi_end_io != end_sync_read)
1538                         continue;
1539                 if (i == first)
1540                         continue;
1541                 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) {
1542                         /* We know that the bi_io_vec layout is the same for
1543                          * both 'first' and 'i', so we just compare them.
1544                          * All vec entries are PAGE_SIZE;
1545                          */
1546                         for (j = 0; j < vcnt; j++)
1547                                 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
1548                                            page_address(tbio->bi_io_vec[j].bv_page),
1549                                            PAGE_SIZE))
1550                                         break;
1551                         if (j == vcnt)
1552                                 continue;
1553                         mddev->resync_mismatches += r10_bio->sectors;
1554                         if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
1555                                 /* Don't fix anything. */
1556                                 continue;
1557                 }
1558                 /* Ok, we need to write this bio, either to correct an
1559                  * inconsistency or to correct an unreadable block.
1560                  * First we need to fixup bv_offset, bv_len and
1561                  * bi_vecs, as the read request might have corrupted these
1562                  */
1563                 tbio->bi_vcnt = vcnt;
1564                 tbio->bi_size = r10_bio->sectors << 9;
1565                 tbio->bi_idx = 0;
1566                 tbio->bi_phys_segments = 0;
1567                 tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
1568                 tbio->bi_flags |= 1 << BIO_UPTODATE;
1569                 tbio->bi_next = NULL;
1570                 tbio->bi_rw = WRITE;
1571                 tbio->bi_private = r10_bio;
1572                 tbio->bi_sector = r10_bio->devs[i].addr;
1573
1574                 for (j=0; j < vcnt ; j++) {
1575                         tbio->bi_io_vec[j].bv_offset = 0;
1576                         tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
1577
1578                         memcpy(page_address(tbio->bi_io_vec[j].bv_page),
1579                                page_address(fbio->bi_io_vec[j].bv_page),
1580                                PAGE_SIZE);
1581                 }
1582                 tbio->bi_end_io = end_sync_write;
1583
1584                 d = r10_bio->devs[i].devnum;
1585                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1586                 atomic_inc(&r10_bio->remaining);
1587                 md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
1588
1589                 tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
1590                 tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
1591                 generic_make_request(tbio);
1592         }
1593
1594 done:
1595         if (atomic_dec_and_test(&r10_bio->remaining)) {
1596                 md_done_sync(mddev, r10_bio->sectors, 1);
1597                 put_buf(r10_bio);
1598         }
1599 }
1600
1601 /*
1602  * Now for the recovery code.
1603  * Recovery happens across physical sectors.
1604  * We recover all non-is_sync drives by finding the virtual address of
1605  * each, and then choose a working drive that also has that virt address.
1606  * There is a separate r10_bio for each non-in_sync drive.
1607  * Only the first two slots are in use. The first for reading,
1608  * The second for writing.
1609  *
1610  */
1611 static void fix_recovery_read_error(r10bio_t *r10_bio)
1612 {
1613         /* We got a read error during recovery.
1614          * We repeat the read in smaller page-sized sections.
1615          * If a read succeeds, write it to the new device or record
1616          * a bad block if we cannot.
1617          * If a read fails, record a bad block on both old and
1618          * new devices.
1619          */
1620         mddev_t *mddev = r10_bio->mddev;
1621         conf_t *conf = mddev->private;
1622         struct bio *bio = r10_bio->devs[0].bio;
1623         sector_t sect = 0;
1624         int sectors = r10_bio->sectors;
1625         int idx = 0;
1626         int dr = r10_bio->devs[0].devnum;
1627         int dw = r10_bio->devs[1].devnum;
1628
1629         while (sectors) {
1630                 int s = sectors;
1631                 mdk_rdev_t *rdev;
1632                 sector_t addr;
1633                 int ok;
1634
1635                 if (s > (PAGE_SIZE>>9))
1636                         s = PAGE_SIZE >> 9;
1637
1638                 rdev = conf->mirrors[dr].rdev;
1639                 addr = r10_bio->devs[0].addr + sect,
1640                 ok = sync_page_io(rdev,
1641                                   addr,
1642                                   s << 9,
1643                                   bio->bi_io_vec[idx].bv_page,
1644                                   READ, false);
1645                 if (ok) {
1646                         rdev = conf->mirrors[dw].rdev;
1647                         addr = r10_bio->devs[1].addr + sect;
1648                         ok = sync_page_io(rdev,
1649                                           addr,
1650                                           s << 9,
1651                                           bio->bi_io_vec[idx].bv_page,
1652                                           WRITE, false);
1653                         if (!ok)
1654                                 set_bit(WriteErrorSeen, &rdev->flags);
1655                 }
1656                 if (!ok) {
1657                         /* We don't worry if we cannot set a bad block -
1658                          * it really is bad so there is no loss in not
1659                          * recording it yet
1660                          */
1661                         rdev_set_badblocks(rdev, addr, s, 0);
1662
1663                         if (rdev != conf->mirrors[dw].rdev) {
1664                                 /* need bad block on destination too */
1665                                 mdk_rdev_t *rdev2 = conf->mirrors[dw].rdev;
1666                                 addr = r10_bio->devs[1].addr + sect;
1667                                 ok = rdev_set_badblocks(rdev2, addr, s, 0);
1668                                 if (!ok) {
1669                                         /* just abort the recovery */
1670                                         printk(KERN_NOTICE
1671                                                "md/raid10:%s: recovery aborted"
1672                                                " due to read error\n",
1673                                                mdname(mddev));
1674
1675                                         conf->mirrors[dw].recovery_disabled
1676                                                 = mddev->recovery_disabled;
1677                                         set_bit(MD_RECOVERY_INTR,
1678                                                 &mddev->recovery);
1679                                         break;
1680                                 }
1681                         }
1682                 }
1683
1684                 sectors -= s;
1685                 sect += s;
1686                 idx++;
1687         }
1688 }
1689
1690 static void recovery_request_write(mddev_t *mddev, r10bio_t *r10_bio)
1691 {
1692         conf_t *conf = mddev->private;
1693         int d;
1694         struct bio *wbio;
1695
1696         if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
1697                 fix_recovery_read_error(r10_bio);
1698                 end_sync_request(r10_bio);
1699                 return;
1700         }
1701
1702         /*
1703          * share the pages with the first bio
1704          * and submit the write request
1705          */
1706         wbio = r10_bio->devs[1].bio;
1707         d = r10_bio->devs[1].devnum;
1708
1709         atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1710         md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
1711         generic_make_request(wbio);
1712 }
1713
1714
1715 /*
1716  * Used by fix_read_error() to decay the per rdev read_errors.
1717  * We halve the read error count for every hour that has elapsed
1718  * since the last recorded read error.
1719  *
1720  */
1721 static void check_decay_read_errors(mddev_t *mddev, mdk_rdev_t *rdev)
1722 {
1723         struct timespec cur_time_mon;
1724         unsigned long hours_since_last;
1725         unsigned int read_errors = atomic_read(&rdev->read_errors);
1726
1727         ktime_get_ts(&cur_time_mon);
1728
1729         if (rdev->last_read_error.tv_sec == 0 &&
1730             rdev->last_read_error.tv_nsec == 0) {
1731                 /* first time we've seen a read error */
1732                 rdev->last_read_error = cur_time_mon;
1733                 return;
1734         }
1735
1736         hours_since_last = (cur_time_mon.tv_sec -
1737                             rdev->last_read_error.tv_sec) / 3600;
1738
1739         rdev->last_read_error = cur_time_mon;
1740
1741         /*
1742          * if hours_since_last is > the number of bits in read_errors
1743          * just set read errors to 0. We do this to avoid
1744          * overflowing the shift of read_errors by hours_since_last.
1745          */
1746         if (hours_since_last >= 8 * sizeof(read_errors))
1747                 atomic_set(&rdev->read_errors, 0);
1748         else
1749                 atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
1750 }
1751
1752 static int r10_sync_page_io(mdk_rdev_t *rdev, sector_t sector,
1753                             int sectors, struct page *page, int rw)
1754 {
1755         sector_t first_bad;
1756         int bad_sectors;
1757
1758         if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
1759             && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
1760                 return -1;
1761         if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
1762                 /* success */
1763                 return 1;
1764         if (rw == WRITE)
1765                 set_bit(WriteErrorSeen, &rdev->flags);
1766         /* need to record an error - either for the block or the device */
1767         if (!rdev_set_badblocks(rdev, sector, sectors, 0))
1768                 md_error(rdev->mddev, rdev);
1769         return 0;
1770 }
1771
1772 /*
1773  * This is a kernel thread which:
1774  *
1775  *      1.      Retries failed read operations on working mirrors.
1776  *      2.      Updates the raid superblock when problems encounter.
1777  *      3.      Performs writes following reads for array synchronising.
1778  */
1779
1780 static void fix_read_error(conf_t *conf, mddev_t *mddev, r10bio_t *r10_bio)
1781 {
1782         int sect = 0; /* Offset from r10_bio->sector */
1783         int sectors = r10_bio->sectors;
1784         mdk_rdev_t*rdev;
1785         int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
1786         int d = r10_bio->devs[r10_bio->read_slot].devnum;
1787
1788         /* still own a reference to this rdev, so it cannot
1789          * have been cleared recently.
1790          */
1791         rdev = conf->mirrors[d].rdev;
1792
1793         if (test_bit(Faulty, &rdev->flags))
1794                 /* drive has already been failed, just ignore any
1795                    more fix_read_error() attempts */
1796                 return;
1797
1798         check_decay_read_errors(mddev, rdev);
1799         atomic_inc(&rdev->read_errors);
1800         if (atomic_read(&rdev->read_errors) > max_read_errors) {
1801                 char b[BDEVNAME_SIZE];
1802                 bdevname(rdev->bdev, b);
1803
1804                 printk(KERN_NOTICE
1805                        "md/raid10:%s: %s: Raid device exceeded "
1806                        "read_error threshold [cur %d:max %d]\n",
1807                        mdname(mddev), b,
1808                        atomic_read(&rdev->read_errors), max_read_errors);
1809                 printk(KERN_NOTICE
1810                        "md/raid10:%s: %s: Failing raid device\n",
1811                        mdname(mddev), b);
1812                 md_error(mddev, conf->mirrors[d].rdev);
1813                 return;
1814         }
1815
1816         while(sectors) {
1817                 int s = sectors;
1818                 int sl = r10_bio->read_slot;
1819                 int success = 0;
1820                 int start;
1821
1822                 if (s > (PAGE_SIZE>>9))
1823                         s = PAGE_SIZE >> 9;
1824
1825                 rcu_read_lock();
1826                 do {
1827                         sector_t first_bad;
1828                         int bad_sectors;
1829
1830                         d = r10_bio->devs[sl].devnum;
1831                         rdev = rcu_dereference(conf->mirrors[d].rdev);
1832                         if (rdev &&
1833                             test_bit(In_sync, &rdev->flags) &&
1834                             is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
1835                                         &first_bad, &bad_sectors) == 0) {
1836                                 atomic_inc(&rdev->nr_pending);
1837                                 rcu_read_unlock();
1838                                 success = sync_page_io(rdev,
1839                                                        r10_bio->devs[sl].addr +
1840                                                        sect,
1841                                                        s<<9,
1842                                                        conf->tmppage, READ, false);
1843                                 rdev_dec_pending(rdev, mddev);
1844                                 rcu_read_lock();
1845                                 if (success)
1846                                         break;
1847                         }
1848                         sl++;
1849                         if (sl == conf->copies)
1850                                 sl = 0;
1851                 } while (!success && sl != r10_bio->read_slot);
1852                 rcu_read_unlock();
1853
1854                 if (!success) {
1855                         /* Cannot read from anywhere, just mark the block
1856                          * as bad on the first device to discourage future
1857                          * reads.
1858                          */
1859                         int dn = r10_bio->devs[r10_bio->read_slot].devnum;
1860                         rdev = conf->mirrors[dn].rdev;
1861
1862                         if (!rdev_set_badblocks(
1863                                     rdev,
1864                                     r10_bio->devs[r10_bio->read_slot].addr
1865                                     + sect,
1866                                     s, 0))
1867                                 md_error(mddev, rdev);
1868                         break;
1869                 }
1870
1871                 start = sl;
1872                 /* write it back and re-read */
1873                 rcu_read_lock();
1874                 while (sl != r10_bio->read_slot) {
1875                         char b[BDEVNAME_SIZE];
1876
1877                         if (sl==0)
1878                                 sl = conf->copies;
1879                         sl--;
1880                         d = r10_bio->devs[sl].devnum;
1881                         rdev = rcu_dereference(conf->mirrors[d].rdev);
1882                         if (!rdev ||
1883                             !test_bit(In_sync, &rdev->flags))
1884                                 continue;
1885
1886                         atomic_inc(&rdev->nr_pending);
1887                         rcu_read_unlock();
1888                         if (r10_sync_page_io(rdev,
1889                                              r10_bio->devs[sl].addr +
1890                                              sect,
1891                                              s<<9, conf->tmppage, WRITE)
1892                             == 0) {
1893                                 /* Well, this device is dead */
1894                                 printk(KERN_NOTICE
1895                                        "md/raid10:%s: read correction "
1896                                        "write failed"
1897                                        " (%d sectors at %llu on %s)\n",
1898                                        mdname(mddev), s,
1899                                        (unsigned long long)(
1900                                                sect + rdev->data_offset),
1901                                        bdevname(rdev->bdev, b));
1902                                 printk(KERN_NOTICE "md/raid10:%s: %s: failing "
1903                                        "drive\n",
1904                                        mdname(mddev),
1905                                        bdevname(rdev->bdev, b));
1906                         }
1907                         rdev_dec_pending(rdev, mddev);
1908                         rcu_read_lock();
1909                 }
1910                 sl = start;
1911                 while (sl != r10_bio->read_slot) {
1912                         char b[BDEVNAME_SIZE];
1913
1914                         if (sl==0)
1915                                 sl = conf->copies;
1916                         sl--;
1917                         d = r10_bio->devs[sl].devnum;
1918                         rdev = rcu_dereference(conf->mirrors[d].rdev);
1919                         if (!rdev ||
1920                             !test_bit(In_sync, &rdev->flags))
1921                                 continue;
1922
1923                         atomic_inc(&rdev->nr_pending);
1924                         rcu_read_unlock();
1925                         switch (r10_sync_page_io(rdev,
1926                                              r10_bio->devs[sl].addr +
1927                                              sect,
1928                                              s<<9, conf->tmppage,
1929                                                  READ)) {
1930                         case 0:
1931                                 /* Well, this device is dead */
1932                                 printk(KERN_NOTICE
1933                                        "md/raid10:%s: unable to read back "
1934                                        "corrected sectors"
1935                                        " (%d sectors at %llu on %s)\n",
1936                                        mdname(mddev), s,
1937                                        (unsigned long long)(
1938                                                sect + rdev->data_offset),
1939                                        bdevname(rdev->bdev, b));
1940                                 printk(KERN_NOTICE "md/raid10:%s: %s: failing "
1941                                        "drive\n",
1942                                        mdname(mddev),
1943                                        bdevname(rdev->bdev, b));
1944                                 break;
1945                         case 1:
1946                                 printk(KERN_INFO
1947                                        "md/raid10:%s: read error corrected"
1948                                        " (%d sectors at %llu on %s)\n",
1949                                        mdname(mddev), s,
1950                                        (unsigned long long)(
1951                                                sect + rdev->data_offset),
1952                                        bdevname(rdev->bdev, b));
1953                                 atomic_add(s, &rdev->corrected_errors);
1954                         }
1955
1956                         rdev_dec_pending(rdev, mddev);
1957                         rcu_read_lock();
1958                 }
1959                 rcu_read_unlock();
1960
1961                 sectors -= s;
1962                 sect += s;
1963         }
1964 }
1965
1966 static void bi_complete(struct bio *bio, int error)
1967 {
1968         complete((struct completion *)bio->bi_private);
1969 }
1970
1971 static int submit_bio_wait(int rw, struct bio *bio)
1972 {
1973         struct completion event;
1974         rw |= REQ_SYNC;
1975
1976         init_completion(&event);
1977         bio->bi_private = &event;
1978         bio->bi_end_io = bi_complete;
1979         submit_bio(rw, bio);
1980         wait_for_completion(&event);
1981
1982         return test_bit(BIO_UPTODATE, &bio->bi_flags);
1983 }
1984
1985 static int narrow_write_error(r10bio_t *r10_bio, int i)
1986 {
1987         struct bio *bio = r10_bio->master_bio;
1988         mddev_t *mddev = r10_bio->mddev;
1989         conf_t *conf = mddev->private;
1990         mdk_rdev_t *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
1991         /* bio has the data to be written to slot 'i' where
1992          * we just recently had a write error.
1993          * We repeatedly clone the bio and trim down to one block,
1994          * then try the write.  Where the write fails we record
1995          * a bad block.
1996          * It is conceivable that the bio doesn't exactly align with
1997          * blocks.  We must handle this.
1998          *
1999          * We currently own a reference to the rdev.
2000          */
2001
2002         int block_sectors;
2003         sector_t sector;
2004         int sectors;
2005         int sect_to_write = r10_bio->sectors;
2006         int ok = 1;
2007
2008         if (rdev->badblocks.shift < 0)
2009                 return 0;
2010
2011         block_sectors = 1 << rdev->badblocks.shift;
2012         sector = r10_bio->sector;
2013         sectors = ((r10_bio->sector + block_sectors)
2014                    & ~(sector_t)(block_sectors - 1))
2015                 - sector;
2016
2017         while (sect_to_write) {
2018                 struct bio *wbio;
2019                 if (sectors > sect_to_write)
2020                         sectors = sect_to_write;
2021                 /* Write at 'sector' for 'sectors' */
2022                 wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
2023                 md_trim_bio(wbio, sector - bio->bi_sector, sectors);
2024                 wbio->bi_sector = (r10_bio->devs[i].addr+
2025                                    rdev->data_offset+
2026                                    (sector - r10_bio->sector));
2027                 wbio->bi_bdev = rdev->bdev;
2028                 if (submit_bio_wait(WRITE, wbio) == 0)
2029                         /* Failure! */
2030                         ok = rdev_set_badblocks(rdev, sector,
2031                                                 sectors, 0)
2032                                 && ok;
2033
2034                 bio_put(wbio);
2035                 sect_to_write -= sectors;
2036                 sector += sectors;
2037                 sectors = block_sectors;
2038         }
2039         return ok;
2040 }
2041
2042 static void handle_read_error(mddev_t *mddev, r10bio_t *r10_bio)
2043 {
2044         int slot = r10_bio->read_slot;
2045         int mirror = r10_bio->devs[slot].devnum;
2046         struct bio *bio;
2047         conf_t *conf = mddev->private;
2048         mdk_rdev_t *rdev;
2049         char b[BDEVNAME_SIZE];
2050         unsigned long do_sync;
2051         int max_sectors;
2052
2053         /* we got a read error. Maybe the drive is bad.  Maybe just
2054          * the block and we can fix it.
2055          * We freeze all other IO, and try reading the block from
2056          * other devices.  When we find one, we re-write
2057          * and check it that fixes the read error.
2058          * This is all done synchronously while the array is
2059          * frozen.
2060          */
2061         if (mddev->ro == 0) {
2062                 freeze_array(conf);
2063                 fix_read_error(conf, mddev, r10_bio);
2064                 unfreeze_array(conf);
2065         }
2066         rdev_dec_pending(conf->mirrors[mirror].rdev, mddev);
2067
2068         bio = r10_bio->devs[slot].bio;
2069         bdevname(bio->bi_bdev, b);
2070         r10_bio->devs[slot].bio =
2071                 mddev->ro ? IO_BLOCKED : NULL;
2072 read_more:
2073         mirror = read_balance(conf, r10_bio, &max_sectors);
2074         if (mirror == -1) {
2075                 printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
2076                        " read error for block %llu\n",
2077                        mdname(mddev), b,
2078                        (unsigned long long)r10_bio->sector);
2079                 raid_end_bio_io(r10_bio);
2080                 bio_put(bio);
2081                 return;
2082         }
2083
2084         do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC);
2085         if (bio)
2086                 bio_put(bio);
2087         slot = r10_bio->read_slot;
2088         rdev = conf->mirrors[mirror].rdev;
2089         printk_ratelimited(
2090                 KERN_ERR
2091                 "md/raid10:%s: %s: redirecting"
2092                 "sector %llu to another mirror\n",
2093                 mdname(mddev),
2094                 bdevname(rdev->bdev, b),
2095                 (unsigned long long)r10_bio->sector);
2096         bio = bio_clone_mddev(r10_bio->master_bio,
2097                               GFP_NOIO, mddev);
2098         md_trim_bio(bio,
2099                     r10_bio->sector - bio->bi_sector,
2100                     max_sectors);
2101         r10_bio->devs[slot].bio = bio;
2102         bio->bi_sector = r10_bio->devs[slot].addr
2103                 + rdev->data_offset;
2104         bio->bi_bdev = rdev->bdev;
2105         bio->bi_rw = READ | do_sync;
2106         bio->bi_private = r10_bio;
2107         bio->bi_end_io = raid10_end_read_request;
2108         if (max_sectors < r10_bio->sectors) {
2109                 /* Drat - have to split this up more */
2110                 struct bio *mbio = r10_bio->master_bio;
2111                 int sectors_handled =
2112                         r10_bio->sector + max_sectors
2113                         - mbio->bi_sector;
2114                 r10_bio->sectors = max_sectors;
2115                 spin_lock_irq(&conf->device_lock);
2116                 if (mbio->bi_phys_segments == 0)
2117                         mbio->bi_phys_segments = 2;
2118                 else
2119                         mbio->bi_phys_segments++;
2120                 spin_unlock_irq(&conf->device_lock);
2121                 generic_make_request(bio);
2122                 bio = NULL;
2123
2124                 r10_bio = mempool_alloc(conf->r10bio_pool,
2125                                         GFP_NOIO);
2126                 r10_bio->master_bio = mbio;
2127                 r10_bio->sectors = (mbio->bi_size >> 9)
2128                         - sectors_handled;
2129                 r10_bio->state = 0;
2130                 set_bit(R10BIO_ReadError,
2131                         &r10_bio->state);
2132                 r10_bio->mddev = mddev;
2133                 r10_bio->sector = mbio->bi_sector
2134                         + sectors_handled;
2135
2136                 goto read_more;
2137         } else
2138                 generic_make_request(bio);
2139 }
2140
2141 static void handle_write_completed(conf_t *conf, r10bio_t *r10_bio)
2142 {
2143         /* Some sort of write request has finished and it
2144          * succeeded in writing where we thought there was a
2145          * bad block.  So forget the bad block.
2146          * Or possibly if failed and we need to record
2147          * a bad block.
2148          */
2149         int m;
2150         mdk_rdev_t *rdev;
2151
2152         if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2153             test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2154                 for (m = 0; m < conf->copies; m++) {
2155                         int dev = r10_bio->devs[m].devnum;
2156                         rdev = conf->mirrors[dev].rdev;
2157                         if (r10_bio->devs[m].bio == NULL)
2158                                 continue;
2159                         if (test_bit(BIO_UPTODATE,
2160                                      &r10_bio->devs[m].bio->bi_flags)) {
2161                                 rdev_clear_badblocks(
2162                                         rdev,
2163                                         r10_bio->devs[m].addr,
2164                                         r10_bio->sectors);
2165                         } else {
2166                                 if (!rdev_set_badblocks(
2167                                             rdev,
2168                                             r10_bio->devs[m].addr,
2169                                             r10_bio->sectors, 0))
2170                                         md_error(conf->mddev, rdev);
2171                         }
2172                 }
2173                 put_buf(r10_bio);
2174         } else {
2175                 for (m = 0; m < conf->copies; m++) {
2176                         int dev = r10_bio->devs[m].devnum;
2177                         struct bio *bio = r10_bio->devs[m].bio;
2178                         rdev = conf->mirrors[dev].rdev;
2179                         if (bio == IO_MADE_GOOD) {
2180                                 rdev_clear_badblocks(
2181                                         rdev,
2182                                         r10_bio->devs[m].addr,
2183                                         r10_bio->sectors);
2184                                 rdev_dec_pending(rdev, conf->mddev);
2185                         } else if (bio != NULL &&
2186                                    !test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2187                                 if (!narrow_write_error(r10_bio, m)) {
2188                                         md_error(conf->mddev, rdev);
2189                                         set_bit(R10BIO_Degraded,
2190                                                 &r10_bio->state);
2191                                 }
2192                                 rdev_dec_pending(rdev, conf->mddev);
2193                         }
2194                 }
2195                 if (test_bit(R10BIO_WriteError,
2196                              &r10_bio->state))
2197                         close_write(r10_bio);
2198                 raid_end_bio_io(r10_bio);
2199         }
2200 }
2201
2202 static void raid10d(mddev_t *mddev)
2203 {
2204         r10bio_t *r10_bio;
2205         unsigned long flags;
2206         conf_t *conf = mddev->private;
2207         struct list_head *head = &conf->retry_list;
2208         struct blk_plug plug;
2209
2210         md_check_recovery(mddev);
2211
2212         blk_start_plug(&plug);
2213         for (;;) {
2214
2215                 flush_pending_writes(conf);
2216
2217                 spin_lock_irqsave(&conf->device_lock, flags);
2218                 if (list_empty(head)) {
2219                         spin_unlock_irqrestore(&conf->device_lock, flags);
2220                         break;
2221                 }
2222                 r10_bio = list_entry(head->prev, r10bio_t, retry_list);
2223                 list_del(head->prev);
2224                 conf->nr_queued--;
2225                 spin_unlock_irqrestore(&conf->device_lock, flags);
2226
2227                 mddev = r10_bio->mddev;
2228                 conf = mddev->private;
2229                 if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2230                     test_bit(R10BIO_WriteError, &r10_bio->state))
2231                         handle_write_completed(conf, r10_bio);
2232                 else if (test_bit(R10BIO_IsSync, &r10_bio->state))
2233                         sync_request_write(mddev, r10_bio);
2234                 else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
2235                         recovery_request_write(mddev, r10_bio);
2236                 else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2237                         handle_read_error(mddev, r10_bio);
2238                 else {
2239                         /* just a partial read to be scheduled from a
2240                          * separate context
2241                          */
2242                         int slot = r10_bio->read_slot;
2243                         generic_make_request(r10_bio->devs[slot].bio);
2244                 }
2245
2246                 cond_resched();
2247                 if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
2248                         md_check_recovery(mddev);
2249         }
2250         blk_finish_plug(&plug);
2251 }
2252
2253
2254 static int init_resync(conf_t *conf)
2255 {
2256         int buffs;
2257
2258         buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2259         BUG_ON(conf->r10buf_pool);
2260         conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
2261         if (!conf->r10buf_pool)
2262                 return -ENOMEM;
2263         conf->next_resync = 0;
2264         return 0;
2265 }
2266
2267 /*
2268  * perform a "sync" on one "block"
2269  *
2270  * We need to make sure that no normal I/O request - particularly write
2271  * requests - conflict with active sync requests.
2272  *
2273  * This is achieved by tracking pending requests and a 'barrier' concept
2274  * that can be installed to exclude normal IO requests.
2275  *
2276  * Resync and recovery are handled very differently.
2277  * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
2278  *
2279  * For resync, we iterate over virtual addresses, read all copies,
2280  * and update if there are differences.  If only one copy is live,
2281  * skip it.
2282  * For recovery, we iterate over physical addresses, read a good
2283  * value for each non-in_sync drive, and over-write.
2284  *
2285  * So, for recovery we may have several outstanding complex requests for a
2286  * given address, one for each out-of-sync device.  We model this by allocating
2287  * a number of r10_bio structures, one for each out-of-sync device.
2288  * As we setup these structures, we collect all bio's together into a list
2289  * which we then process collectively to add pages, and then process again
2290  * to pass to generic_make_request.
2291  *
2292  * The r10_bio structures are linked using a borrowed master_bio pointer.
2293  * This link is counted in ->remaining.  When the r10_bio that points to NULL
2294  * has its remaining count decremented to 0, the whole complex operation
2295  * is complete.
2296  *
2297  */
2298
2299 static sector_t sync_request(mddev_t *mddev, sector_t sector_nr,
2300                              int *skipped, int go_faster)
2301 {
2302         conf_t *conf = mddev->private;
2303         r10bio_t *r10_bio;
2304         struct bio *biolist = NULL, *bio;
2305         sector_t max_sector, nr_sectors;
2306         int i;
2307         int max_sync;
2308         sector_t sync_blocks;
2309         sector_t sectors_skipped = 0;
2310         int chunks_skipped = 0;
2311
2312         if (!conf->r10buf_pool)
2313                 if (init_resync(conf))
2314                         return 0;
2315
2316  skipped:
2317         max_sector = mddev->dev_sectors;
2318         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
2319                 max_sector = mddev->resync_max_sectors;
2320         if (sector_nr >= max_sector) {
2321                 /* If we aborted, we need to abort the
2322                  * sync on the 'current' bitmap chucks (there can
2323                  * be several when recovering multiple devices).
2324                  * as we may have started syncing it but not finished.
2325                  * We can find the current address in
2326                  * mddev->curr_resync, but for recovery,
2327                  * we need to convert that to several
2328                  * virtual addresses.
2329                  */
2330                 if (mddev->curr_resync < max_sector) { /* aborted */
2331                         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
2332                                 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2333                                                 &sync_blocks, 1);
2334                         else for (i=0; i<conf->raid_disks; i++) {
2335                                 sector_t sect =
2336                                         raid10_find_virt(conf, mddev->curr_resync, i);
2337                                 bitmap_end_sync(mddev->bitmap, sect,
2338                                                 &sync_blocks, 1);
2339                         }
2340                 } else /* completed sync */
2341                         conf->fullsync = 0;
2342
2343                 bitmap_close_sync(mddev->bitmap);
2344                 close_sync(conf);
2345                 *skipped = 1;
2346                 return sectors_skipped;
2347         }
2348         if (chunks_skipped >= conf->raid_disks) {
2349                 /* if there has been nothing to do on any drive,
2350                  * then there is nothing to do at all..
2351                  */
2352                 *skipped = 1;
2353                 return (max_sector - sector_nr) + sectors_skipped;
2354         }
2355
2356         if (max_sector > mddev->resync_max)
2357                 max_sector = mddev->resync_max; /* Don't do IO beyond here */
2358
2359         /* make sure whole request will fit in a chunk - if chunks
2360          * are meaningful
2361          */
2362         if (conf->near_copies < conf->raid_disks &&
2363             max_sector > (sector_nr | conf->chunk_mask))
2364                 max_sector = (sector_nr | conf->chunk_mask) + 1;
2365         /*
2366          * If there is non-resync activity waiting for us then
2367          * put in a delay to throttle resync.
2368          */
2369         if (!go_faster && conf->nr_waiting)
2370                 msleep_interruptible(1000);
2371
2372         /* Again, very different code for resync and recovery.
2373          * Both must result in an r10bio with a list of bios that
2374          * have bi_end_io, bi_sector, bi_bdev set,
2375          * and bi_private set to the r10bio.
2376          * For recovery, we may actually create several r10bios
2377          * with 2 bios in each, that correspond to the bios in the main one.
2378          * In this case, the subordinate r10bios link back through a
2379          * borrowed master_bio pointer, and the counter in the master
2380          * includes a ref from each subordinate.
2381          */
2382         /* First, we decide what to do and set ->bi_end_io
2383          * To end_sync_read if we want to read, and
2384          * end_sync_write if we will want to write.
2385          */
2386
2387         max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
2388         if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2389                 /* recovery... the complicated one */
2390                 int j;
2391                 r10_bio = NULL;
2392
2393                 for (i=0 ; i<conf->raid_disks; i++) {
2394                         int still_degraded;
2395                         r10bio_t *rb2;
2396                         sector_t sect;
2397                         int must_sync;
2398                         int any_working;
2399
2400                         if (conf->mirrors[i].rdev == NULL ||
2401                             test_bit(In_sync, &conf->mirrors[i].rdev->flags))
2402                                 continue;
2403
2404                         still_degraded = 0;
2405                         /* want to reconstruct this device */
2406                         rb2 = r10_bio;
2407                         sect = raid10_find_virt(conf, sector_nr, i);
2408                         /* Unless we are doing a full sync, we only need
2409                          * to recover the block if it is set in the bitmap
2410                          */
2411                         must_sync = bitmap_start_sync(mddev->bitmap, sect,
2412                                                       &sync_blocks, 1);
2413                         if (sync_blocks < max_sync)
2414                                 max_sync = sync_blocks;
2415                         if (!must_sync &&
2416                             !conf->fullsync) {
2417                                 /* yep, skip the sync_blocks here, but don't assume
2418                                  * that there will never be anything to do here
2419                                  */
2420                                 chunks_skipped = -1;
2421                                 continue;
2422                         }
2423
2424                         r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
2425                         raise_barrier(conf, rb2 != NULL);
2426                         atomic_set(&r10_bio->remaining, 0);
2427
2428                         r10_bio->master_bio = (struct bio*)rb2;
2429                         if (rb2)
2430                                 atomic_inc(&rb2->remaining);
2431                         r10_bio->mddev = mddev;
2432                         set_bit(R10BIO_IsRecover, &r10_bio->state);
2433                         r10_bio->sector = sect;
2434
2435                         raid10_find_phys(conf, r10_bio);
2436
2437                         /* Need to check if the array will still be
2438                          * degraded
2439                          */
2440                         for (j=0; j<conf->raid_disks; j++)
2441                                 if (conf->mirrors[j].rdev == NULL ||
2442                                     test_bit(Faulty, &conf->mirrors[j].rdev->flags)) {
2443                                         still_degraded = 1;
2444                                         break;
2445                                 }
2446
2447                         must_sync = bitmap_start_sync(mddev->bitmap, sect,
2448                                                       &sync_blocks, still_degraded);
2449
2450                         any_working = 0;
2451                         for (j=0; j<conf->copies;j++) {
2452                                 int k;
2453                                 int d = r10_bio->devs[j].devnum;
2454                                 sector_t from_addr, to_addr;
2455                                 mdk_rdev_t *rdev;
2456                                 sector_t sector, first_bad;
2457                                 int bad_sectors;
2458                                 if (!conf->mirrors[d].rdev ||
2459                                     !test_bit(In_sync, &conf->mirrors[d].rdev->flags))
2460                                         continue;
2461                                 /* This is where we read from */
2462                                 any_working = 1;
2463                                 rdev = conf->mirrors[d].rdev;
2464                                 sector = r10_bio->devs[j].addr;
2465
2466                                 if (is_badblock(rdev, sector, max_sync,
2467                                                 &first_bad, &bad_sectors)) {
2468                                         if (first_bad > sector)
2469                                                 max_sync = first_bad - sector;
2470                                         else {
2471                                                 bad_sectors -= (sector
2472                                                                 - first_bad);
2473                                                 if (max_sync > bad_sectors)
2474                                                         max_sync = bad_sectors;
2475                                                 continue;
2476                                         }
2477                                 }
2478                                 bio = r10_bio->devs[0].bio;
2479                                 bio->bi_next = biolist;
2480                                 biolist = bio;
2481                                 bio->bi_private = r10_bio;
2482                                 bio->bi_end_io = end_sync_read;
2483                                 bio->bi_rw = READ;
2484                                 from_addr = r10_bio->devs[j].addr;
2485                                 bio->bi_sector = from_addr +
2486                                         conf->mirrors[d].rdev->data_offset;
2487                                 bio->bi_bdev = conf->mirrors[d].rdev->bdev;
2488                                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2489                                 atomic_inc(&r10_bio->remaining);
2490                                 /* and we write to 'i' */
2491
2492                                 for (k=0; k<conf->copies; k++)
2493                                         if (r10_bio->devs[k].devnum == i)
2494                                                 break;
2495                                 BUG_ON(k == conf->copies);
2496                                 bio = r10_bio->devs[1].bio;
2497                                 bio->bi_next = biolist;
2498                                 biolist = bio;
2499                                 bio->bi_private = r10_bio;
2500                                 bio->bi_end_io = end_sync_write;
2501                                 bio->bi_rw = WRITE;
2502                                 to_addr = r10_bio->devs[k].addr;
2503                                 bio->bi_sector = to_addr +
2504                                         conf->mirrors[i].rdev->data_offset;
2505                                 bio->bi_bdev = conf->mirrors[i].rdev->bdev;
2506
2507                                 r10_bio->devs[0].devnum = d;
2508                                 r10_bio->devs[0].addr = from_addr;
2509                                 r10_bio->devs[1].devnum = i;
2510                                 r10_bio->devs[1].addr = to_addr;
2511
2512                                 break;
2513                         }
2514                         if (j == conf->copies) {
2515                                 /* Cannot recover, so abort the recovery or
2516                                  * record a bad block */
2517                                 put_buf(r10_bio);
2518                                 if (rb2)
2519                                         atomic_dec(&rb2->remaining);
2520                                 r10_bio = rb2;
2521                                 if (any_working) {
2522                                         /* problem is that there are bad blocks
2523                                          * on other device(s)
2524                                          */
2525                                         int k;
2526                                         for (k = 0; k < conf->copies; k++)
2527                                                 if (r10_bio->devs[k].devnum == i)
2528                                                         break;
2529                                         if (!rdev_set_badblocks(
2530                                                     conf->mirrors[i].rdev,
2531                                                     r10_bio->devs[k].addr,
2532                                                     max_sync, 0))
2533                                                 any_working = 0;
2534                                 }
2535                                 if (!any_working)  {
2536                                         if (!test_and_set_bit(MD_RECOVERY_INTR,
2537                                                               &mddev->recovery))
2538                                                 printk(KERN_INFO "md/raid10:%s: insufficient "
2539                                                        "working devices for recovery.\n",
2540                                                        mdname(mddev));
2541                                         conf->mirrors[i].recovery_disabled
2542                                                 = mddev->recovery_disabled;
2543                                 }
2544                                 break;
2545                         }
2546                 }
2547                 if (biolist == NULL) {
2548                         while (r10_bio) {
2549                                 r10bio_t *rb2 = r10_bio;
2550                                 r10_bio = (r10bio_t*) rb2->master_bio;
2551                                 rb2->master_bio = NULL;
2552                                 put_buf(rb2);
2553                         }
2554                         goto giveup;
2555                 }
2556         } else {
2557                 /* resync. Schedule a read for every block at this virt offset */
2558                 int count = 0;
2559
2560                 bitmap_cond_end_sync(mddev->bitmap, sector_nr);
2561
2562                 if (!bitmap_start_sync(mddev->bitmap, sector_nr,
2563                                        &sync_blocks, mddev->degraded) &&
2564                     !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
2565                                                  &mddev->recovery)) {
2566                         /* We can skip this block */
2567                         *skipped = 1;
2568                         return sync_blocks + sectors_skipped;
2569                 }
2570                 if (sync_blocks < max_sync)
2571                         max_sync = sync_blocks;
2572                 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
2573
2574                 r10_bio->mddev = mddev;
2575                 atomic_set(&r10_bio->remaining, 0);
2576                 raise_barrier(conf, 0);
2577                 conf->next_resync = sector_nr;
2578
2579                 r10_bio->master_bio = NULL;
2580                 r10_bio->sector = sector_nr;
2581                 set_bit(R10BIO_IsSync, &r10_bio->state);
2582                 raid10_find_phys(conf, r10_bio);
2583                 r10_bio->sectors = (sector_nr | conf->chunk_mask) - sector_nr +1;
2584
2585                 for (i=0; i<conf->copies; i++) {
2586                         int d = r10_bio->devs[i].devnum;
2587                         sector_t first_bad, sector;
2588                         int bad_sectors;
2589
2590                         bio = r10_bio->devs[i].bio;
2591                         bio->bi_end_io = NULL;
2592                         clear_bit(BIO_UPTODATE, &bio->bi_flags);
2593                         if (conf->mirrors[d].rdev == NULL ||
2594                             test_bit(Faulty, &conf->mirrors[d].rdev->flags))
2595                                 continue;
2596                         sector = r10_bio->devs[i].addr;
2597                         if (is_badblock(conf->mirrors[d].rdev,
2598                                         sector, max_sync,
2599                                         &first_bad, &bad_sectors)) {
2600                                 if (first_bad > sector)
2601                                         max_sync = first_bad - sector;
2602                                 else {
2603                                         bad_sectors -= (sector - first_bad);
2604                                         if (max_sync > bad_sectors)
2605                                                 max_sync = max_sync;
2606                                         continue;
2607                                 }
2608                         }
2609                         atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2610                         atomic_inc(&r10_bio->remaining);
2611                         bio->bi_next = biolist;
2612                         biolist = bio;
2613                         bio->bi_private = r10_bio;
2614                         bio->bi_end_io = end_sync_read;
2615                         bio->bi_rw = READ;
2616                         bio->bi_sector = sector +
2617                                 conf->mirrors[d].rdev->data_offset;
2618                         bio->bi_bdev = conf->mirrors[d].rdev->bdev;
2619                         count++;
2620                 }
2621
2622                 if (count < 2) {
2623                         for (i=0; i<conf->copies; i++) {
2624                                 int d = r10_bio->devs[i].devnum;
2625                                 if (r10_bio->devs[i].bio->bi_end_io)
2626                                         rdev_dec_pending(conf->mirrors[d].rdev,
2627                                                          mddev);
2628                         }
2629                         put_buf(r10_bio);
2630                         biolist = NULL;
2631                         goto giveup;
2632                 }
2633         }
2634
2635         for (bio = biolist; bio ; bio=bio->bi_next) {
2636
2637                 bio->bi_flags &= ~(BIO_POOL_MASK - 1);
2638                 if (bio->bi_end_io)
2639                         bio->bi_flags |= 1 << BIO_UPTODATE;
2640                 bio->bi_vcnt = 0;
2641                 bio->bi_idx = 0;
2642                 bio->bi_phys_segments = 0;
2643                 bio->bi_size = 0;
2644         }
2645
2646         nr_sectors = 0;
2647         if (sector_nr + max_sync < max_sector)
2648                 max_sector = sector_nr + max_sync;
2649         do {
2650                 struct page *page;
2651                 int len = PAGE_SIZE;
2652                 if (sector_nr + (len>>9) > max_sector)
2653                         len = (max_sector - sector_nr) << 9;
2654                 if (len == 0)
2655                         break;
2656                 for (bio= biolist ; bio ; bio=bio->bi_next) {
2657                         struct bio *bio2;
2658                         page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
2659                         if (bio_add_page(bio, page, len, 0))
2660                                 continue;
2661
2662                         /* stop here */
2663                         bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
2664                         for (bio2 = biolist;
2665                              bio2 && bio2 != bio;
2666                              bio2 = bio2->bi_next) {
2667                                 /* remove last page from this bio */
2668                                 bio2->bi_vcnt--;
2669                                 bio2->bi_size -= len;
2670                                 bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
2671                         }
2672                         goto bio_full;
2673                 }
2674                 nr_sectors += len>>9;
2675                 sector_nr += len>>9;
2676         } while (biolist->bi_vcnt < RESYNC_PAGES);
2677  bio_full:
2678         r10_bio->sectors = nr_sectors;
2679
2680         while (biolist) {
2681                 bio = biolist;
2682                 biolist = biolist->bi_next;
2683
2684                 bio->bi_next = NULL;
2685                 r10_bio = bio->bi_private;
2686                 r10_bio->sectors = nr_sectors;
2687
2688                 if (bio->bi_end_io == end_sync_read) {
2689                         md_sync_acct(bio->bi_bdev, nr_sectors);
2690                         generic_make_request(bio);
2691                 }
2692         }
2693
2694         if (sectors_skipped)
2695                 /* pretend they weren't skipped, it makes
2696                  * no important difference in this case
2697                  */
2698                 md_done_sync(mddev, sectors_skipped, 1);
2699
2700         return sectors_skipped + nr_sectors;
2701  giveup:
2702         /* There is nowhere to write, so all non-sync
2703          * drives must be failed or in resync, all drives
2704          * have a bad block, so try the next chunk...
2705          */
2706         if (sector_nr + max_sync < max_sector)
2707                 max_sector = sector_nr + max_sync;
2708
2709         sectors_skipped += (max_sector - sector_nr);
2710         chunks_skipped ++;
2711         sector_nr = max_sector;
2712         goto skipped;
2713 }
2714
2715 static sector_t
2716 raid10_size(mddev_t *mddev, sector_t sectors, int raid_disks)
2717 {
2718         sector_t size;
2719         conf_t *conf = mddev->private;
2720
2721         if (!raid_disks)
2722                 raid_disks = conf->raid_disks;
2723         if (!sectors)
2724                 sectors = conf->dev_sectors;
2725
2726         size = sectors >> conf->chunk_shift;
2727         sector_div(size, conf->far_copies);
2728         size = size * raid_disks;
2729         sector_div(size, conf->near_copies);
2730
2731         return size << conf->chunk_shift;
2732 }
2733
2734
2735 static conf_t *setup_conf(mddev_t *mddev)
2736 {
2737         conf_t *conf = NULL;
2738         int nc, fc, fo;
2739         sector_t stride, size;
2740         int err = -EINVAL;
2741
2742         if (mddev->new_chunk_sectors < (PAGE_SIZE >> 9) ||
2743             !is_power_of_2(mddev->new_chunk_sectors)) {
2744                 printk(KERN_ERR "md/raid10:%s: chunk size must be "
2745                        "at least PAGE_SIZE(%ld) and be a power of 2.\n",
2746                        mdname(mddev), PAGE_SIZE);
2747                 goto out;
2748         }
2749
2750         nc = mddev->new_layout & 255;
2751         fc = (mddev->new_layout >> 8) & 255;
2752         fo = mddev->new_layout & (1<<16);
2753
2754         if ((nc*fc) <2 || (nc*fc) > mddev->raid_disks ||
2755             (mddev->new_layout >> 17)) {
2756                 printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
2757                        mdname(mddev), mddev->new_layout);
2758                 goto out;
2759         }
2760
2761         err = -ENOMEM;
2762         conf = kzalloc(sizeof(conf_t), GFP_KERNEL);
2763         if (!conf)
2764                 goto out;
2765
2766         conf->mirrors = kzalloc(sizeof(struct mirror_info)*mddev->raid_disks,
2767                                 GFP_KERNEL);
2768         if (!conf->mirrors)
2769                 goto out;
2770
2771         conf->tmppage = alloc_page(GFP_KERNEL);
2772         if (!conf->tmppage)
2773                 goto out;
2774
2775
2776         conf->raid_disks = mddev->raid_disks;
2777         conf->near_copies = nc;
2778         conf->far_copies = fc;
2779         conf->copies = nc*fc;
2780         conf->far_offset = fo;
2781         conf->chunk_mask = mddev->new_chunk_sectors - 1;
2782         conf->chunk_shift = ffz(~mddev->new_chunk_sectors);
2783
2784         conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
2785                                            r10bio_pool_free, conf);
2786         if (!conf->r10bio_pool)
2787                 goto out;
2788
2789         size = mddev->dev_sectors >> conf->chunk_shift;
2790         sector_div(size, fc);
2791         size = size * conf->raid_disks;
2792         sector_div(size, nc);
2793         /* 'size' is now the number of chunks in the array */
2794         /* calculate "used chunks per device" in 'stride' */
2795         stride = size * conf->copies;
2796
2797         /* We need to round up when dividing by raid_disks to
2798          * get the stride size.
2799          */
2800         stride += conf->raid_disks - 1;
2801         sector_div(stride, conf->raid_disks);
2802
2803         conf->dev_sectors = stride << conf->chunk_shift;
2804
2805         if (fo)
2806                 stride = 1;
2807         else
2808                 sector_div(stride, fc);
2809         conf->stride = stride << conf->chunk_shift;
2810
2811
2812         spin_lock_init(&conf->device_lock);
2813         INIT_LIST_HEAD(&conf->retry_list);
2814
2815         spin_lock_init(&conf->resync_lock);
2816         init_waitqueue_head(&conf->wait_barrier);
2817
2818         conf->thread = md_register_thread(raid10d, mddev, NULL);
2819         if (!conf->thread)
2820                 goto out;
2821
2822         conf->mddev = mddev;
2823         return conf;
2824
2825  out:
2826         printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
2827                mdname(mddev));
2828         if (conf) {
2829                 if (conf->r10bio_pool)
2830                         mempool_destroy(conf->r10bio_pool);
2831                 kfree(conf->mirrors);
2832                 safe_put_page(conf->tmppage);
2833                 kfree(conf);
2834         }
2835         return ERR_PTR(err);
2836 }
2837
2838 static int run(mddev_t *mddev)
2839 {
2840         conf_t *conf;
2841         int i, disk_idx, chunk_size;
2842         mirror_info_t *disk;
2843         mdk_rdev_t *rdev;
2844         sector_t size;
2845
2846         /*
2847          * copy the already verified devices into our private RAID10
2848          * bookkeeping area. [whatever we allocate in run(),
2849          * should be freed in stop()]
2850          */
2851
2852         if (mddev->private == NULL) {
2853                 conf = setup_conf(mddev);
2854                 if (IS_ERR(conf))
2855                         return PTR_ERR(conf);
2856                 mddev->private = conf;
2857         }
2858         conf = mddev->private;
2859         if (!conf)
2860                 goto out;
2861
2862         mddev->thread = conf->thread;
2863         conf->thread = NULL;
2864
2865         chunk_size = mddev->chunk_sectors << 9;
2866         blk_queue_io_min(mddev->queue, chunk_size);
2867         if (conf->raid_disks % conf->near_copies)
2868                 blk_queue_io_opt(mddev->queue, chunk_size * conf->raid_disks);
2869         else
2870                 blk_queue_io_opt(mddev->queue, chunk_size *
2871                                  (conf->raid_disks / conf->near_copies));
2872
2873         list_for_each_entry(rdev, &mddev->disks, same_set) {
2874
2875                 disk_idx = rdev->raid_disk;
2876                 if (disk_idx >= conf->raid_disks
2877                     || disk_idx < 0)
2878                         continue;
2879                 disk = conf->mirrors + disk_idx;
2880
2881                 disk->rdev = rdev;
2882                 disk_stack_limits(mddev->gendisk, rdev->bdev,
2883                                   rdev->data_offset << 9);
2884                 /* as we don't honour merge_bvec_fn, we must never risk
2885                  * violating it, so limit max_segments to 1 lying
2886                  * within a single page.
2887                  */
2888                 if (rdev->bdev->bd_disk->queue->merge_bvec_fn) {
2889                         blk_queue_max_segments(mddev->queue, 1);
2890                         blk_queue_segment_boundary(mddev->queue,
2891                                                    PAGE_CACHE_SIZE - 1);
2892                 }
2893
2894                 disk->head_position = 0;
2895         }
2896         /* need to check that every block has at least one working mirror */
2897         if (!enough(conf, -1)) {
2898                 printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n",
2899                        mdname(mddev));
2900                 goto out_free_conf;
2901         }
2902
2903         mddev->degraded = 0;
2904         for (i = 0; i < conf->raid_disks; i++) {
2905
2906                 disk = conf->mirrors + i;
2907
2908                 if (!disk->rdev ||
2909                     !test_bit(In_sync, &disk->rdev->flags)) {
2910                         disk->head_position = 0;
2911                         mddev->degraded++;
2912                         if (disk->rdev)
2913                                 conf->fullsync = 1;
2914                 }
2915         }
2916
2917         if (mddev->recovery_cp != MaxSector)
2918                 printk(KERN_NOTICE "md/raid10:%s: not clean"
2919                        " -- starting background reconstruction\n",
2920                        mdname(mddev));
2921         printk(KERN_INFO
2922                 "md/raid10:%s: active with %d out of %d devices\n",
2923                 mdname(mddev), conf->raid_disks - mddev->degraded,
2924                 conf->raid_disks);
2925         /*
2926          * Ok, everything is just fine now
2927          */
2928         mddev->dev_sectors = conf->dev_sectors;
2929         size = raid10_size(mddev, 0, 0);
2930         md_set_array_sectors(mddev, size);
2931         mddev->resync_max_sectors = size;
2932
2933         mddev->queue->backing_dev_info.congested_fn = raid10_congested;
2934         mddev->queue->backing_dev_info.congested_data = mddev;
2935
2936         /* Calculate max read-ahead size.
2937          * We need to readahead at least twice a whole stripe....
2938          * maybe...
2939          */
2940         {
2941                 int stripe = conf->raid_disks *
2942                         ((mddev->chunk_sectors << 9) / PAGE_SIZE);
2943                 stripe /= conf->near_copies;
2944                 if (mddev->queue->backing_dev_info.ra_pages < 2* stripe)
2945                         mddev->queue->backing_dev_info.ra_pages = 2* stripe;
2946         }
2947
2948         if (conf->near_copies < conf->raid_disks)
2949                 blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec);
2950
2951         if (md_integrity_register(mddev))
2952                 goto out_free_conf;
2953
2954         return 0;
2955
2956 out_free_conf:
2957         md_unregister_thread(mddev->thread);
2958         if (conf->r10bio_pool)
2959                 mempool_destroy(conf->r10bio_pool);
2960         safe_put_page(conf->tmppage);
2961         kfree(conf->mirrors);
2962         kfree(conf);
2963         mddev->private = NULL;
2964 out:
2965         return -EIO;
2966 }
2967
2968 static int stop(mddev_t *mddev)
2969 {
2970         conf_t *conf = mddev->private;
2971
2972         raise_barrier(conf, 0);
2973         lower_barrier(conf);
2974
2975         md_unregister_thread(mddev->thread);
2976         mddev->thread = NULL;
2977         blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
2978         if (conf->r10bio_pool)
2979                 mempool_destroy(conf->r10bio_pool);
2980         kfree(conf->mirrors);
2981         kfree(conf);
2982         mddev->private = NULL;
2983         return 0;
2984 }
2985
2986 static void raid10_quiesce(mddev_t *mddev, int state)
2987 {
2988         conf_t *conf = mddev->private;
2989
2990         switch(state) {
2991         case 1:
2992                 raise_barrier(conf, 0);
2993                 break;
2994         case 0:
2995                 lower_barrier(conf);
2996                 break;
2997         }
2998 }
2999
3000 static void *raid10_takeover_raid0(mddev_t *mddev)
3001 {
3002         mdk_rdev_t *rdev;
3003         conf_t *conf;
3004
3005         if (mddev->degraded > 0) {
3006                 printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n",
3007                        mdname(mddev));
3008                 return ERR_PTR(-EINVAL);
3009         }
3010
3011         /* Set new parameters */
3012         mddev->new_level = 10;
3013         /* new layout: far_copies = 1, near_copies = 2 */
3014         mddev->new_layout = (1<<8) + 2;
3015         mddev->new_chunk_sectors = mddev->chunk_sectors;
3016         mddev->delta_disks = mddev->raid_disks;
3017         mddev->raid_disks *= 2;
3018         /* make sure it will be not marked as dirty */
3019         mddev->recovery_cp = MaxSector;
3020
3021         conf = setup_conf(mddev);
3022         if (!IS_ERR(conf)) {
3023                 list_for_each_entry(rdev, &mddev->disks, same_set)
3024                         if (rdev->raid_disk >= 0)
3025                                 rdev->new_raid_disk = rdev->raid_disk * 2;
3026                 conf->barrier = 1;
3027         }
3028
3029         return conf;
3030 }
3031
3032 static void *raid10_takeover(mddev_t *mddev)
3033 {
3034         struct raid0_private_data *raid0_priv;
3035
3036         /* raid10 can take over:
3037          *  raid0 - providing it has only two drives
3038          */
3039         if (mddev->level == 0) {
3040                 /* for raid0 takeover only one zone is supported */
3041                 raid0_priv = mddev->private;
3042                 if (raid0_priv->nr_strip_zones > 1) {
3043                         printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0"
3044                                " with more than one zone.\n",
3045                                mdname(mddev));
3046                         return ERR_PTR(-EINVAL);
3047                 }
3048                 return raid10_takeover_raid0(mddev);
3049         }
3050         return ERR_PTR(-EINVAL);
3051 }
3052
3053 static struct mdk_personality raid10_personality =
3054 {
3055         .name           = "raid10",
3056         .level          = 10,
3057         .owner          = THIS_MODULE,
3058         .make_request   = make_request,
3059         .run            = run,
3060         .stop           = stop,
3061         .status         = status,
3062         .error_handler  = error,
3063         .hot_add_disk   = raid10_add_disk,
3064         .hot_remove_disk= raid10_remove_disk,
3065         .spare_active   = raid10_spare_active,
3066         .sync_request   = sync_request,
3067         .quiesce        = raid10_quiesce,
3068         .size           = raid10_size,
3069         .takeover       = raid10_takeover,
3070 };
3071
3072 static int __init raid_init(void)
3073 {
3074         return register_md_personality(&raid10_personality);
3075 }
3076
3077 static void raid_exit(void)
3078 {
3079         unregister_md_personality(&raid10_personality);
3080 }
3081
3082 module_init(raid_init);
3083 module_exit(raid_exit);
3084 MODULE_LICENSE("GPL");
3085 MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
3086 MODULE_ALIAS("md-personality-9"); /* RAID10 */
3087 MODULE_ALIAS("md-raid10");
3088 MODULE_ALIAS("md-level-10");