fs/btrfs/scrub.c

   1 /*
   2  * Copyright (C) 2011 STRATO.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/blkdev.h>
  20 #include "ctree.h"
  21 #include "volumes.h"
  22 #include "disk-io.h"
  23 #include "ordered-data.h"
  24
  25 /*
  26  * This is only the first step towards a full-features scrub. It reads all
  27  * extent and super block and verifies the checksums. In case a bad checksum
  28  * is found or the extent cannot be read, good data will be written back if
  29  * any can be found.
  30  *
  31  * Future enhancements:
  32  *  - To enhance the performance, better read-ahead strategies for the
  33  *    extent-tree can be employed.
  34  *  - In case an unrepairable extent is encountered, track which files are
  35  *    affected and report them
  36  *  - In case of a read error on files with nodatasum, map the file and read
  37  *    the extent to trigger a writeback of the good copy
  38  *  - track and record media errors, throw out bad devices
  39  *  - add a mode to also read unallocated space
  40  *  - make the prefetch cancellable
  41  */
  42
  43 struct scrub_bio;
  44 struct scrub_page;
  45 struct scrub_dev;
  46 static void scrub_bio_end_io(struct bio *bio, int err);
  47 static void scrub_checksum(struct btrfs_work *work);
  48 static int scrub_checksum_data(struct scrub_dev *sdev,
  49                                struct scrub_page *spag, void *buffer);
  50 static int scrub_checksum_tree_block(struct scrub_dev *sdev,
  51                                      struct scrub_page *spag, u64 logical,
  52                                      void *buffer);
  53 static int scrub_checksum_super(struct scrub_bio *sbio, void *buffer);
  54 static int scrub_fixup_check(struct scrub_bio *sbio, int ix);
  55 static void scrub_fixup_end_io(struct bio *bio, int err);
  56 static int scrub_fixup_io(int rw, struct block_device *bdev, sector_t sector,
  57                           struct page *page);
  58 static void scrub_fixup(struct scrub_bio *sbio, int ix);
  59
  60 #define SCRUB_PAGES_PER_BIO     16      /* 64k per bio */
  61 #define SCRUB_BIOS_PER_DEV      16      /* 1 MB per device in flight */
  62
  63 struct scrub_page {
  64         u64                     flags;  /* extent flags */
  65         u64                     generation;
  66         u64                     mirror_num;
  67         int                     have_csum;
  68         u8                      csum[BTRFS_CSUM_SIZE];
  69 };
  70
  71 struct scrub_bio {
  72         int                     index;
  73         struct scrub_dev        *sdev;
  74         struct bio              *bio;
  75         int                     err;
  76         u64                     logical;
  77         u64                     physical;
  78         struct scrub_page       spag[SCRUB_PAGES_PER_BIO];
  79         u64                     count;
  80         int                     next_free;
  81         struct btrfs_work       work;
  82 };
  83
  84 struct scrub_dev {
  85         struct scrub_bio        *bios[SCRUB_BIOS_PER_DEV];
  86         struct btrfs_device     *dev;
  87         int                     first_free;
  88         int                     curr;
  89         atomic_t                in_flight;
  90         spinlock_t              list_lock;
  91         wait_queue_head_t       list_wait;
  92         u16                     csum_size;
  93         struct list_head        csum_list;
  94         atomic_t                cancel_req;
  95         int                     readonly;
  96         /*
  97          * statistics
  98          */
  99         struct btrfs_scrub_progress stat;
 100         spinlock_t              stat_lock;
 101 };
 102
 103 static void scrub_free_csums(struct scrub_dev *sdev)
 104 {
 105         while (!list_empty(&sdev->csum_list)) {
 106                 struct btrfs_ordered_sum *sum;
 107                 sum = list_first_entry(&sdev->csum_list,
 108                                        struct btrfs_ordered_sum, list);
 109                 list_del(&sum->list);
 110                 kfree(sum);
 111         }
 112 }
 113
 114 static void scrub_free_bio(struct bio *bio)
 115 {
 116         int i;
 117         struct page *last_page = NULL;
 118
 119         if (!bio)
 120                 return;
 121
 122         for (i = 0; i < bio->bi_vcnt; ++i) {
 123                 if (bio->bi_io_vec[i].bv_page == last_page)
 124                         continue;
 125                 last_page = bio->bi_io_vec[i].bv_page;
 126                 __free_page(last_page);
 127         }
 128         bio_put(bio);
 129 }
 130
 131 static noinline_for_stack void scrub_free_dev(struct scrub_dev *sdev)
 132 {
 133         int i;
 134
 135         if (!sdev)
 136                 return;
 137
 138         for (i = 0; i < SCRUB_BIOS_PER_DEV; ++i) {
 139                 struct scrub_bio *sbio = sdev->bios[i];
 140
 141                 if (!sbio)
 142                         break;
 143
 144                 scrub_free_bio(sbio->bio);
 145                 kfree(sbio);
 146         }
 147
 148         scrub_free_csums(sdev);
 149         kfree(sdev);
 150 }
 151
 152 static noinline_for_stack
 153 struct scrub_dev *scrub_setup_dev(struct btrfs_device *dev)
 154 {
 155         struct scrub_dev *sdev;
 156         int             i;
 157         struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
 158
 159         sdev = kzalloc(sizeof(*sdev), GFP_NOFS);
 160         if (!sdev)
 161                 goto nomem;
 162         sdev->dev = dev;
 163         for (i = 0; i < SCRUB_BIOS_PER_DEV; ++i) {
 164                 struct scrub_bio *sbio;
 165
 166                 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
 167                 if (!sbio)
 168                         goto nomem;
 169                 sdev->bios[i] = sbio;
 170
 171                 sbio->index = i;
 172                 sbio->sdev = sdev;
 173                 sbio->count = 0;
 174                 sbio->work.func = scrub_checksum;
 175
 176                 if (i != SCRUB_BIOS_PER_DEV-1)
 177                         sdev->bios[i]->next_free = i + 1;
 178                  else
 179                         sdev->bios[i]->next_free = -1;
 180         }
 181         sdev->first_free = 0;
 182         sdev->curr = -1;
 183         atomic_set(&sdev->in_flight, 0);
 184         atomic_set(&sdev->cancel_req, 0);
 185         sdev->csum_size = btrfs_super_csum_size(&fs_info->super_copy);
 186         INIT_LIST_HEAD(&sdev->csum_list);
 187
 188         spin_lock_init(&sdev->list_lock);
 189         spin_lock_init(&sdev->stat_lock);
 190         init_waitqueue_head(&sdev->list_wait);
 191         return sdev;
 192
 193 nomem:
 194         scrub_free_dev(sdev);
 195         return ERR_PTR(-ENOMEM);
 196 }
 197
 198 /*
 199  * scrub_recheck_error gets called when either verification of the page
 200  * failed or the bio failed to read, e.g. with EIO. In the latter case,
 201  * recheck_error gets called for every page in the bio, even though only
 202  * one may be bad
 203  */
 204 static void scrub_recheck_error(struct scrub_bio *sbio, int ix)
 205 {
 206         if (sbio->err) {
 207                 if (scrub_fixup_io(READ, sbio->sdev->dev->bdev,
 208                                    (sbio->physical + ix * PAGE_SIZE) >> 9,
 209                                    sbio->bio->bi_io_vec[ix].bv_page) == 0) {
 210                         if (scrub_fixup_check(sbio, ix) == 0)
 211                                 return;
 212                 }
 213         }
 214
 215         scrub_fixup(sbio, ix);
 216 }
 217
 218 static int scrub_fixup_check(struct scrub_bio *sbio, int ix)
 219 {
 220         int ret = 1;
 221         struct page *page;
 222         void *buffer;
 223         u64 flags = sbio->spag[ix].flags;
 224
 225         page = sbio->bio->bi_io_vec[ix].bv_page;
 226         buffer = kmap_atomic(page, KM_USER0);
 227         if (flags & BTRFS_EXTENT_FLAG_DATA) {
 228                 ret = scrub_checksum_data(sbio->sdev,
 229                                           sbio->spag + ix, buffer);
 230         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 231                 ret = scrub_checksum_tree_block(sbio->sdev,
 232                                                 sbio->spag + ix,
 233                                                 sbio->logical + ix * PAGE_SIZE,
 234                                                 buffer);
 235         } else {
 236                 WARN_ON(1);
 237         }
 238         kunmap_atomic(buffer, KM_USER0);
 239
 240         return ret;
 241 }
 242
 243 static void scrub_fixup_end_io(struct bio *bio, int err)
 244 {
 245         complete((struct completion *)bio->bi_private);
 246 }
 247
 248 static void scrub_fixup(struct scrub_bio *sbio, int ix)
 249 {
 250         struct scrub_dev *sdev = sbio->sdev;
 251         struct btrfs_fs_info *fs_info = sdev->dev->dev_root->fs_info;
 252         struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
 253         struct btrfs_multi_bio *multi = NULL;
 254         u64 logical = sbio->logical + ix * PAGE_SIZE;
 255         u64 length;
 256         int i;
 257         int ret;
 258         DECLARE_COMPLETION_ONSTACK(complete);
 259
 260         if ((sbio->spag[ix].flags & BTRFS_EXTENT_FLAG_DATA) &&
 261             (sbio->spag[ix].have_csum == 0)) {
 262                 /*
 263                  * nodatasum, don't try to fix anything
 264                  * FIXME: we can do better, open the inode and trigger a
 265                  * writeback
 266                  */
 267                 goto uncorrectable;
 268         }
 269
 270         length = PAGE_SIZE;
 271         ret = btrfs_map_block(map_tree, REQ_WRITE, logical, &length,
 272                               &multi, 0);
 273         if (ret || !multi || length < PAGE_SIZE) {
 274                 printk(KERN_ERR
 275                        "scrub_fixup: btrfs_map_block failed us for %llu\n",
 276                        (unsigned long long)logical);
 277                 WARN_ON(1);
 278                 return;
 279         }
 280
 281         if (multi->num_stripes == 1)
 282                 /* there aren't any replicas */
 283                 goto uncorrectable;
 284
 285         /*
 286          * first find a good copy
 287          */
 288         for (i = 0; i < multi->num_stripes; ++i) {
 289                 if (i == sbio->spag[ix].mirror_num)
 290                         continue;
 291
 292                 if (scrub_fixup_io(READ, multi->stripes[i].dev->bdev,
 293                                    multi->stripes[i].physical >> 9,
 294                                    sbio->bio->bi_io_vec[ix].bv_page)) {
 295                         /* I/O-error, this is not a good copy */
 296                         continue;
 297                 }
 298
 299                 if (scrub_fixup_check(sbio, ix) == 0)
 300                         break;
 301         }
 302         if (i == multi->num_stripes)
 303                 goto uncorrectable;
 304
 305         if (!sdev->readonly) {
 306                 /*
 307                  * bi_io_vec[ix].bv_page now contains good data, write it back
 308                  */
 309                 if (scrub_fixup_io(WRITE, sdev->dev->bdev,
 310                                    (sbio->physical + ix * PAGE_SIZE) >> 9,
 311                                    sbio->bio->bi_io_vec[ix].bv_page)) {
 312                         /* I/O-error, writeback failed, give up */
 313                         goto uncorrectable;
 314                 }
 315         }
 316
 317         kfree(multi);
 318         spin_lock(&sdev->stat_lock);
 319         ++sdev->stat.corrected_errors;
 320         spin_unlock(&sdev->stat_lock);
 321
 322         if (printk_ratelimit())
 323                 printk(KERN_ERR "btrfs: fixed up at %llu\n",
 324                        (unsigned long long)logical);
 325         return;
 326
 327 uncorrectable:
 328         kfree(multi);
 329         spin_lock(&sdev->stat_lock);
 330         ++sdev->stat.uncorrectable_errors;
 331         spin_unlock(&sdev->stat_lock);
 332
 333         if (printk_ratelimit())
 334                 printk(KERN_ERR "btrfs: unable to fixup at %llu\n",
 335                          (unsigned long long)logical);
 336 }
 337
 338 static int scrub_fixup_io(int rw, struct block_device *bdev, sector_t sector,
 339                          struct page *page)
 340 {
 341         struct bio *bio = NULL;
 342         int ret;
 343         DECLARE_COMPLETION_ONSTACK(complete);
 344
 345         bio = bio_alloc(GFP_NOFS, 1);
 346         bio->bi_bdev = bdev;
 347         bio->bi_sector = sector;
 348         bio_add_page(bio, page, PAGE_SIZE, 0);
 349         bio->bi_end_io = scrub_fixup_end_io;
 350         bio->bi_private = &complete;
 351         submit_bio(rw, bio);
 352
 353         /* this will also unplug the queue */
 354         wait_for_completion(&complete);
 355
 356         ret = !test_bit(BIO_UPTODATE, &bio->bi_flags);
 357         bio_put(bio);
 358         return ret;
 359 }
 360
 361 static void scrub_bio_end_io(struct bio *bio, int err)
 362 {
 363         struct scrub_bio *sbio = bio->bi_private;
 364         struct scrub_dev *sdev = sbio->sdev;
 365         struct btrfs_fs_info *fs_info = sdev->dev->dev_root->fs_info;
 366
 367         sbio->err = err;
 368         sbio->bio = bio;
 369
 370         btrfs_queue_worker(&fs_info->scrub_workers, &sbio->work);
 371 }
 372
 373 static void scrub_checksum(struct btrfs_work *work)
 374 {
 375         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
 376         struct scrub_dev *sdev = sbio->sdev;
 377         struct page *page;
 378         void *buffer;
 379         int i;
 380         u64 flags;
 381         u64 logical;
 382         int ret;
 383
 384         if (sbio->err) {
 385                 for (i = 0; i < sbio->count; ++i)
 386                         scrub_recheck_error(sbio, i);
 387
 388                 sbio->bio->bi_flags &= ~(BIO_POOL_MASK - 1);
 389                 sbio->bio->bi_flags |= 1 << BIO_UPTODATE;
 390                 sbio->bio->bi_phys_segments = 0;
 391                 sbio->bio->bi_idx = 0;
 392
 393                 for (i = 0; i < sbio->count; i++) {
 394                         struct bio_vec *bi;
 395                         bi = &sbio->bio->bi_io_vec[i];
 396                         bi->bv_offset = 0;
 397                         bi->bv_len = PAGE_SIZE;
 398                 }
 399
 400                 spin_lock(&sdev->stat_lock);
 401                 ++sdev->stat.read_errors;
 402                 spin_unlock(&sdev->stat_lock);
 403                 goto out;
 404         }
 405         for (i = 0; i < sbio->count; ++i) {
 406                 page = sbio->bio->bi_io_vec[i].bv_page;
 407                 buffer = kmap_atomic(page, KM_USER0);
 408                 flags = sbio->spag[i].flags;
 409                 logical = sbio->logical + i * PAGE_SIZE;
 410                 ret = 0;
 411                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
 412                         ret = scrub_checksum_data(sdev, sbio->spag + i, buffer);
 413                 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 414                         ret = scrub_checksum_tree_block(sdev, sbio->spag + i,
 415                                                         logical, buffer);
 416                 } else if (flags & BTRFS_EXTENT_FLAG_SUPER) {
 417                         BUG_ON(i);
 418                         (void)scrub_checksum_super(sbio, buffer);
 419                 } else {
 420                         WARN_ON(1);
 421                 }
 422                 kunmap_atomic(buffer, KM_USER0);
 423                 if (ret)
 424                         scrub_recheck_error(sbio, i);
 425         }
 426
 427 out:
 428         scrub_free_bio(sbio->bio);
 429         sbio->bio = NULL;
 430         spin_lock(&sdev->list_lock);
 431         sbio->next_free = sdev->first_free;
 432         sdev->first_free = sbio->index;
 433         spin_unlock(&sdev->list_lock);
 434         atomic_dec(&sdev->in_flight);
 435         wake_up(&sdev->list_wait);
 436 }
 437
 438 static int scrub_checksum_data(struct scrub_dev *sdev,
 439                                struct scrub_page *spag, void *buffer)
 440 {
 441         u8 csum[BTRFS_CSUM_SIZE];
 442         u32 crc = ~(u32)0;
 443         int fail = 0;
 444         struct btrfs_root *root = sdev->dev->dev_root;
 445
 446         if (!spag->have_csum)
 447                 return 0;
 448
 449         crc = btrfs_csum_data(root, buffer, crc, PAGE_SIZE);
 450         btrfs_csum_final(crc, csum);
 451         if (memcmp(csum, spag->csum, sdev->csum_size))
 452                 fail = 1;
 453
 454         spin_lock(&sdev->stat_lock);
 455         ++sdev->stat.data_extents_scrubbed;
 456         sdev->stat.data_bytes_scrubbed += PAGE_SIZE;
 457         if (fail)
 458                 ++sdev->stat.csum_errors;
 459         spin_unlock(&sdev->stat_lock);
 460
 461         return fail;
 462 }
 463
 464 static int scrub_checksum_tree_block(struct scrub_dev *sdev,
 465                                      struct scrub_page *spag, u64 logical,
 466                                      void *buffer)
 467 {
 468         struct btrfs_header *h;
 469         struct btrfs_root *root = sdev->dev->dev_root;
 470         struct btrfs_fs_info *fs_info = root->fs_info;
 471         u8 csum[BTRFS_CSUM_SIZE];
 472         u32 crc = ~(u32)0;
 473         int fail = 0;
 474         int crc_fail = 0;
 475
 476         /*
 477          * we don't use the getter functions here, as we
 478          * a) don't have an extent buffer and
 479          * b) the page is already kmapped
 480          */
 481         h = (struct btrfs_header *)buffer;
 482
 483         if (logical != le64_to_cpu(h->bytenr))
 484                 ++fail;
 485
 486         if (spag->generation != le64_to_cpu(h->generation))
 487                 ++fail;
 488
 489         if (memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
 490                 ++fail;
 491
 492         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
 493                    BTRFS_UUID_SIZE))
 494                 ++fail;
 495
 496         crc = btrfs_csum_data(root, buffer + BTRFS_CSUM_SIZE, crc,
 497                               PAGE_SIZE - BTRFS_CSUM_SIZE);
 498         btrfs_csum_final(crc, csum);
 499         if (memcmp(csum, h->csum, sdev->csum_size))
 500                 ++crc_fail;
 501
 502         spin_lock(&sdev->stat_lock);
 503         ++sdev->stat.tree_extents_scrubbed;
 504         sdev->stat.tree_bytes_scrubbed += PAGE_SIZE;
 505         if (crc_fail)
 506                 ++sdev->stat.csum_errors;
 507         if (fail)
 508                 ++sdev->stat.verify_errors;
 509         spin_unlock(&sdev->stat_lock);
 510
 511         return fail || crc_fail;
 512 }
 513
 514 static int scrub_checksum_super(struct scrub_bio *sbio, void *buffer)
 515 {
 516         struct btrfs_super_block *s;
 517         u64 logical;
 518         struct scrub_dev *sdev = sbio->sdev;
 519         struct btrfs_root *root = sdev->dev->dev_root;
 520         struct btrfs_fs_info *fs_info = root->fs_info;
 521         u8 csum[BTRFS_CSUM_SIZE];
 522         u32 crc = ~(u32)0;
 523         int fail = 0;
 524
 525         s = (struct btrfs_super_block *)buffer;
 526         logical = sbio->logical;
 527
 528         if (logical != le64_to_cpu(s->bytenr))
 529                 ++fail;
 530
 531         if (sbio->spag[0].generation != le64_to_cpu(s->generation))
 532                 ++fail;
 533
 534         if (memcmp(s->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
 535                 ++fail;
 536
 537         crc = btrfs_csum_data(root, buffer + BTRFS_CSUM_SIZE, crc,
 538                               PAGE_SIZE - BTRFS_CSUM_SIZE);
 539         btrfs_csum_final(crc, csum);
 540         if (memcmp(csum, s->csum, sbio->sdev->csum_size))
 541                 ++fail;
 542
 543         if (fail) {
 544                 /*
 545                  * if we find an error in a super block, we just report it.
 546                  * They will get written with the next transaction commit
 547                  * anyway
 548                  */
 549                 spin_lock(&sdev->stat_lock);
 550                 ++sdev->stat.super_errors;
 551                 spin_unlock(&sdev->stat_lock);
 552         }
 553
 554         return fail;
 555 }
 556
 557 static int scrub_submit(struct scrub_dev *sdev)
 558 {
 559         struct scrub_bio *sbio;
 560         struct bio *bio;
 561         int i;
 562
 563         if (sdev->curr == -1)
 564                 return 0;
 565
 566         sbio = sdev->bios[sdev->curr];
 567
 568         bio = bio_alloc(GFP_NOFS, sbio->count);
 569         if (!bio)
 570                 goto nomem;
 571
 572         bio->bi_private = sbio;
 573         bio->bi_end_io = scrub_bio_end_io;
 574         bio->bi_bdev = sdev->dev->bdev;
 575         bio->bi_sector = sbio->physical >> 9;
 576
 577         for (i = 0; i < sbio->count; ++i) {
 578                 struct page *page;
 579                 int ret;
 580
 581                 page = alloc_page(GFP_NOFS);
 582                 if (!page)
 583                         goto nomem;
 584
 585                 ret = bio_add_page(bio, page, PAGE_SIZE, 0);
 586                 if (!ret) {
 587                         __free_page(page);
 588                         goto nomem;
 589                 }
 590         }
 591
 592         sbio->err = 0;
 593         sdev->curr = -1;
 594         atomic_inc(&sdev->in_flight);
 595
 596         submit_bio(READ, bio);
 597
 598         return 0;
 599
 600 nomem:
 601         scrub_free_bio(bio);
 602
 603         return -ENOMEM;
 604 }
 605
 606 static int scrub_page(struct scrub_dev *sdev, u64 logical, u64 len,
 607                       u64 physical, u64 flags, u64 gen, u64 mirror_num,
 608                       u8 *csum, int force)
 609 {
 610         struct scrub_bio *sbio;
 611
 612 again:
 613         /*
 614          * grab a fresh bio or wait for one to become available
 615          */
 616         while (sdev->curr == -1) {
 617                 spin_lock(&sdev->list_lock);
 618                 sdev->curr = sdev->first_free;
 619                 if (sdev->curr != -1) {
 620                         sdev->first_free = sdev->bios[sdev->curr]->next_free;
 621                         sdev->bios[sdev->curr]->next_free = -1;
 622                         sdev->bios[sdev->curr]->count = 0;
 623                         spin_unlock(&sdev->list_lock);
 624                 } else {
 625                         spin_unlock(&sdev->list_lock);
 626                         wait_event(sdev->list_wait, sdev->first_free != -1);
 627                 }
 628         }
 629         sbio = sdev->bios[sdev->curr];
 630         if (sbio->count == 0) {
 631                 sbio->physical = physical;
 632                 sbio->logical = logical;
 633         } else if (sbio->physical + sbio->count * PAGE_SIZE != physical ||
 634                    sbio->logical + sbio->count * PAGE_SIZE != logical) {
 635                 int ret;
 636
 637                 ret = scrub_submit(sdev);
 638                 if (ret)
 639                         return ret;
 640                 goto again;
 641         }
 642         sbio->spag[sbio->count].flags = flags;
 643         sbio->spag[sbio->count].generation = gen;
 644         sbio->spag[sbio->count].have_csum = 0;
 645         sbio->spag[sbio->count].mirror_num = mirror_num;
 646         if (csum) {
 647                 sbio->spag[sbio->count].have_csum = 1;
 648                 memcpy(sbio->spag[sbio->count].csum, csum, sdev->csum_size);
 649         }
 650         ++sbio->count;
 651         if (sbio->count == SCRUB_PAGES_PER_BIO || force) {
 652                 int ret;
 653
 654                 ret = scrub_submit(sdev);
 655                 if (ret)
 656                         return ret;
 657         }
 658
 659         return 0;
 660 }
 661
 662 static int scrub_find_csum(struct scrub_dev *sdev, u64 logical, u64 len,
 663                            u8 *csum)
 664 {
 665         struct btrfs_ordered_sum *sum = NULL;
 666         int ret = 0;
 667         unsigned long i;
 668         unsigned long num_sectors;
 669         u32 sectorsize = sdev->dev->dev_root->sectorsize;
 670
 671         while (!list_empty(&sdev->csum_list)) {
 672                 sum = list_first_entry(&sdev->csum_list,
 673                                        struct btrfs_ordered_sum, list);
 674                 if (sum->bytenr > logical)
 675                         return 0;
 676                 if (sum->bytenr + sum->len > logical)
 677                         break;
 678
 679                 ++sdev->stat.csum_discards;
 680                 list_del(&sum->list);
 681                 kfree(sum);
 682                 sum = NULL;
 683         }
 684         if (!sum)
 685                 return 0;
 686
 687         num_sectors = sum->len / sectorsize;
 688         for (i = 0; i < num_sectors; ++i) {
 689                 if (sum->sums[i].bytenr == logical) {
 690                         memcpy(csum, &sum->sums[i].sum, sdev->csum_size);
 691                         ret = 1;
 692                         break;
 693                 }
 694         }
 695         if (ret && i == num_sectors - 1) {
 696                 list_del(&sum->list);
 697                 kfree(sum);
 698         }
 699         return ret;
 700 }
 701
 702 /* scrub extent tries to collect up to 64 kB for each bio */
 703 static int scrub_extent(struct scrub_dev *sdev, u64 logical, u64 len,
 704                         u64 physical, u64 flags, u64 gen, u64 mirror_num)
 705 {
 706         int ret;
 707         u8 csum[BTRFS_CSUM_SIZE];
 708
 709         while (len) {
 710                 u64 l = min_t(u64, len, PAGE_SIZE);
 711                 int have_csum = 0;
 712
 713                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
 714                         /* push csums to sbio */
 715                         have_csum = scrub_find_csum(sdev, logical, l, csum);
 716                         if (have_csum == 0)
 717                                 ++sdev->stat.no_csum;
 718                 }
 719                 ret = scrub_page(sdev, logical, l, physical, flags, gen,
 720                                  mirror_num, have_csum ? csum : NULL, 0);
 721                 if (ret)
 722                         return ret;
 723                 len -= l;
 724                 logical += l;
 725                 physical += l;
 726         }
 727         return 0;
 728 }
 729
 730 static noinline_for_stack int scrub_stripe(struct scrub_dev *sdev,
 731         struct map_lookup *map, int num, u64 base, u64 length)
 732 {
 733         struct btrfs_path *path;
 734         struct btrfs_fs_info *fs_info = sdev->dev->dev_root->fs_info;
 735         struct btrfs_root *root = fs_info->extent_root;
 736         struct btrfs_root *csum_root = fs_info->csum_root;
 737         struct btrfs_extent_item *extent;
 738         struct blk_plug plug;
 739         u64 flags;
 740         int ret;
 741         int slot;
 742         int i;
 743         u64 nstripes;
 744         int start_stripe;
 745         struct extent_buffer *l;
 746         struct btrfs_key key;
 747         u64 physical;
 748         u64 logical;
 749         u64 generation;
 750         u64 mirror_num;
 751
 752         u64 increment = map->stripe_len;
 753         u64 offset;
 754
 755         nstripes = length;
 756         offset = 0;
 757         do_div(nstripes, map->stripe_len);
 758         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
 759                 offset = map->stripe_len * num;
 760                 increment = map->stripe_len * map->num_stripes;
 761                 mirror_num = 0;
 762         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
 763                 int factor = map->num_stripes / map->sub_stripes;
 764                 offset = map->stripe_len * (num / map->sub_stripes);
 765                 increment = map->stripe_len * factor;
 766                 mirror_num = num % map->sub_stripes;
 767         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
 768                 increment = map->stripe_len;
 769                 mirror_num = num % map->num_stripes;
 770         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
 771                 increment = map->stripe_len;
 772                 mirror_num = num % map->num_stripes;
 773         } else {
 774                 increment = map->stripe_len;
 775                 mirror_num = 0;
 776         }
 777
 778         path = btrfs_alloc_path();
 779         if (!path)
 780                 return -ENOMEM;
 781
 782         path->reada = 2;
 783         path->search_commit_root = 1;
 784         path->skip_locking = 1;
 785
 786         /*
 787          * find all extents for each stripe and just read them to get
 788          * them into the page cache
 789          * FIXME: we can do better. build a more intelligent prefetching
 790          */
 791         logical = base + offset;
 792         physical = map->stripes[num].physical;
 793         ret = 0;
 794         for (i = 0; i < nstripes; ++i) {
 795                 key.objectid = logical;
 796                 key.type = BTRFS_EXTENT_ITEM_KEY;
 797                 key.offset = (u64)0;
 798
 799                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 800                 if (ret < 0)
 801                         goto out_noplug;
 802
 803                 /*
 804                  * we might miss half an extent here, but that doesn't matter,
 805                  * as it's only the prefetch
 806                  */
 807                 while (1) {
 808                         l = path->nodes[0];
 809                         slot = path->slots[0];
 810                         if (slot >= btrfs_header_nritems(l)) {
 811                                 ret = btrfs_next_leaf(root, path);
 812                                 if (ret == 0)
 813                                         continue;
 814                                 if (ret < 0)
 815                                         goto out_noplug;
 816
 817                                 break;
 818                         }
 819                         btrfs_item_key_to_cpu(l, &key, slot);
 820
 821                         if (key.objectid >= logical + map->stripe_len)
 822                                 break;
 823
 824                         path->slots[0]++;
 825                 }
 826                 btrfs_release_path(path);
 827                 logical += increment;
 828                 physical += map->stripe_len;
 829                 cond_resched();
 830         }
 831
 832         /*
 833          * collect all data csums for the stripe to avoid seeking during
 834          * the scrub. This might currently (crc32) end up to be about 1MB
 835          */
 836         start_stripe = 0;
 837         blk_start_plug(&plug);
 838 again:
 839         logical = base + offset + start_stripe * increment;
 840         for (i = start_stripe; i < nstripes; ++i) {
 841                 ret = btrfs_lookup_csums_range(csum_root, logical,
 842                                                logical + map->stripe_len - 1,
 843                                                &sdev->csum_list, 1);
 844                 if (ret)
 845                         goto out;
 846
 847                 logical += increment;
 848                 cond_resched();
 849         }
 850         /*
 851          * now find all extents for each stripe and scrub them
 852          */
 853         logical = base + offset + start_stripe * increment;
 854         physical = map->stripes[num].physical + start_stripe * map->stripe_len;
 855         ret = 0;
 856         for (i = start_stripe; i < nstripes; ++i) {
 857                 /*
 858                  * canceled?
 859                  */
 860                 if (atomic_read(&fs_info->scrub_cancel_req) ||
 861                     atomic_read(&sdev->cancel_req)) {
 862                         ret = -ECANCELED;
 863                         goto out;
 864                 }
 865                 /*
 866                  * check to see if we have to pause
 867                  */
 868                 if (atomic_read(&fs_info->scrub_pause_req)) {
 869                         /* push queued extents */
 870                         scrub_submit(sdev);
 871                         wait_event(sdev->list_wait,
 872                                    atomic_read(&sdev->in_flight) == 0);
 873                         atomic_inc(&fs_info->scrubs_paused);
 874                         wake_up(&fs_info->scrub_pause_wait);
 875                         mutex_lock(&fs_info->scrub_lock);
 876                         while (atomic_read(&fs_info->scrub_pause_req)) {
 877                                 mutex_unlock(&fs_info->scrub_lock);
 878                                 wait_event(fs_info->scrub_pause_wait,
 879                                    atomic_read(&fs_info->scrub_pause_req) == 0);
 880                                 mutex_lock(&fs_info->scrub_lock);
 881                         }
 882                         atomic_dec(&fs_info->scrubs_paused);
 883                         mutex_unlock(&fs_info->scrub_lock);
 884                         wake_up(&fs_info->scrub_pause_wait);
 885                         scrub_free_csums(sdev);
 886                         start_stripe = i;
 887                         goto again;
 888                 }
 889
 890                 key.objectid = logical;
 891                 key.type = BTRFS_EXTENT_ITEM_KEY;
 892                 key.offset = (u64)0;
 893
 894                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 895                 if (ret < 0)
 896                         goto out;
 897                 if (ret > 0) {
 898                         ret = btrfs_previous_item(root, path, 0,
 899                                                   BTRFS_EXTENT_ITEM_KEY);
 900                         if (ret < 0)
 901                                 goto out;
 902                         if (ret > 0) {
 903                                 /* there's no smaller item, so stick with the
 904                                  * larger one */
 905                                 btrfs_release_path(path);
 906                                 ret = btrfs_search_slot(NULL, root, &key,
 907                                                         path, 0, 0);
 908                                 if (ret < 0)
 909                                         goto out;
 910                         }
 911                 }
 912
 913                 while (1) {
 914                         l = path->nodes[0];
 915                         slot = path->slots[0];
 916                         if (slot >= btrfs_header_nritems(l)) {
 917                                 ret = btrfs_next_leaf(root, path);
 918                                 if (ret == 0)
 919                                         continue;
 920                                 if (ret < 0)
 921                                         goto out;
 922
 923                                 break;
 924                         }
 925                         btrfs_item_key_to_cpu(l, &key, slot);
 926
 927                         if (key.objectid + key.offset <= logical)
 928                                 goto next;
 929
 930                         if (key.objectid >= logical + map->stripe_len)
 931                                 break;
 932
 933                         if (btrfs_key_type(&key) != BTRFS_EXTENT_ITEM_KEY)
 934                                 goto next;
 935
 936                         extent = btrfs_item_ptr(l, slot,
 937                                                 struct btrfs_extent_item);
 938                         flags = btrfs_extent_flags(l, extent);
 939                         generation = btrfs_extent_generation(l, extent);
 940
 941                         if (key.objectid < logical &&
 942                             (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
 943                                 printk(KERN_ERR
 944                                        "btrfs scrub: tree block %llu spanning "
 945                                        "stripes, ignored. logical=%llu\n",
 946                                        (unsigned long long)key.objectid,
 947                                        (unsigned long long)logical);
 948                                 goto next;
 949                         }
 950
 951                         /*
 952                          * trim extent to this stripe
 953                          */
 954                         if (key.objectid < logical) {
 955                                 key.offset -= logical - key.objectid;
 956                                 key.objectid = logical;
 957                         }
 958                         if (key.objectid + key.offset >
 959                             logical + map->stripe_len) {
 960                                 key.offset = logical + map->stripe_len -
 961                                              key.objectid;
 962                         }
 963
 964                         ret = scrub_extent(sdev, key.objectid, key.offset,
 965                                            key.objectid - logical + physical,
 966                                            flags, generation, mirror_num);
 967                         if (ret)
 968                                 goto out;
 969
 970 next:
 971                         path->slots[0]++;
 972                 }
 973                 btrfs_release_path(path);
 974                 logical += increment;
 975                 physical += map->stripe_len;
 976                 spin_lock(&sdev->stat_lock);
 977                 sdev->stat.last_physical = physical;
 978                 spin_unlock(&sdev->stat_lock);
 979         }
 980         /* push queued extents */
 981         scrub_submit(sdev);
 982
 983 out:
 984         blk_finish_plug(&plug);
 985 out_noplug:
 986         btrfs_free_path(path);
 987         return ret < 0 ? ret : 0;
 988 }
 989
 990 static noinline_for_stack int scrub_chunk(struct scrub_dev *sdev,
 991         u64 chunk_tree, u64 chunk_objectid, u64 chunk_offset, u64 length)
 992 {
 993         struct btrfs_mapping_tree *map_tree =
 994                 &sdev->dev->dev_root->fs_info->mapping_tree;
 995         struct map_lookup *map;
 996         struct extent_map *em;
 997         int i;
 998         int ret = -EINVAL;
 999
1000         read_lock(&map_tree->map_tree.lock);
1001         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
1002         read_unlock(&map_tree->map_tree.lock);
1003
1004         if (!em)
1005                 return -EINVAL;
1006
1007         map = (struct map_lookup *)em->bdev;
1008         if (em->start != chunk_offset)
1009                 goto out;
1010
1011         if (em->len < length)
1012                 goto out;
1013
1014         for (i = 0; i < map->num_stripes; ++i) {
1015                 if (map->stripes[i].dev == sdev->dev) {
1016                         ret = scrub_stripe(sdev, map, i, chunk_offset, length);
1017                         if (ret)
1018                                 goto out;
1019                 }
1020         }
1021 out:
1022         free_extent_map(em);
1023
1024         return ret;
1025 }
1026
1027 static noinline_for_stack
1028 int scrub_enumerate_chunks(struct scrub_dev *sdev, u64 start, u64 end)
1029 {
1030         struct btrfs_dev_extent *dev_extent = NULL;
1031         struct btrfs_path *path;
1032         struct btrfs_root *root = sdev->dev->dev_root;
1033         struct btrfs_fs_info *fs_info = root->fs_info;
1034         u64 length;
1035         u64 chunk_tree;
1036         u64 chunk_objectid;
1037         u64 chunk_offset;
1038         int ret;
1039         int slot;
1040         struct extent_buffer *l;
1041         struct btrfs_key key;
1042         struct btrfs_key found_key;
1043         struct btrfs_block_group_cache *cache;
1044
1045         path = btrfs_alloc_path();
1046         if (!path)
1047                 return -ENOMEM;
1048
1049         path->reada = 2;
1050         path->search_commit_root = 1;
1051         path->skip_locking = 1;
1052
1053         key.objectid = sdev->dev->devid;
1054         key.offset = 0ull;
1055         key.type = BTRFS_DEV_EXTENT_KEY;
1056
1057
1058         while (1) {
1059                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1060                 if (ret < 0)
1061                         break;
1062                 if (ret > 0) {
1063                         if (path->slots[0] >=
1064                             btrfs_header_nritems(path->nodes[0])) {
1065                                 ret = btrfs_next_leaf(root, path);
1066                                 if (ret)
1067                                         break;
1068                         }
1069                 }
1070
1071                 l = path->nodes[0];
1072                 slot = path->slots[0];
1073
1074                 btrfs_item_key_to_cpu(l, &found_key, slot);
1075
1076                 if (found_key.objectid != sdev->dev->devid)
1077                         break;
1078
1079                 if (btrfs_key_type(&found_key) != BTRFS_DEV_EXTENT_KEY)
1080                         break;
1081
1082                 if (found_key.offset >= end)
1083                         break;
1084
1085                 if (found_key.offset < key.offset)
1086                         break;
1087
1088                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1089                 length = btrfs_dev_extent_length(l, dev_extent);
1090
1091                 if (found_key.offset + length <= start) {
1092                         key.offset = found_key.offset + length;
1093                         btrfs_release_path(path);
1094                         continue;
1095                 }
1096
1097                 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
1098                 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
1099                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
1100
1101                 /*
1102                  * get a reference on the corresponding block group to prevent
1103                  * the chunk from going away while we scrub it
1104                  */
1105                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
1106                 if (!cache) {
1107                         ret = -ENOENT;
1108                         break;
1109                 }
1110                 ret = scrub_chunk(sdev, chunk_tree, chunk_objectid,
1111                                   chunk_offset, length);
1112                 btrfs_put_block_group(cache);
1113                 if (ret)
1114                         break;
1115
1116                 key.offset = found_key.offset + length;
1117                 btrfs_release_path(path);
1118         }
1119
1120         btrfs_free_path(path);
1121
1122         /*
1123          * ret can still be 1 from search_slot or next_leaf,
1124          * that's not an error
1125          */
1126         return ret < 0 ? ret : 0;
1127 }
1128
1129 static noinline_for_stack int scrub_supers(struct scrub_dev *sdev)
1130 {
1131         int     i;
1132         u64     bytenr;
1133         u64     gen;
1134         int     ret;
1135         struct btrfs_device *device = sdev->dev;
1136         struct btrfs_root *root = device->dev_root;
1137
1138         gen = root->fs_info->last_trans_committed;
1139
1140         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
1141                 bytenr = btrfs_sb_offset(i);
1142                 if (bytenr + BTRFS_SUPER_INFO_SIZE >= device->total_bytes)
1143                         break;
1144
1145                 ret = scrub_page(sdev, bytenr, PAGE_SIZE, bytenr,
1146                                  BTRFS_EXTENT_FLAG_SUPER, gen, i, NULL, 1);
1147                 if (ret)
1148                         return ret;
1149         }
1150         wait_event(sdev->list_wait, atomic_read(&sdev->in_flight) == 0);
1151
1152         return 0;
1153 }
1154
1155 /*
1156  * get a reference count on fs_info->scrub_workers. start worker if necessary
1157  */
1158 static noinline_for_stack int scrub_workers_get(struct btrfs_root *root)
1159 {
1160         struct btrfs_fs_info *fs_info = root->fs_info;
1161
1162         mutex_lock(&fs_info->scrub_lock);
1163         if (fs_info->scrub_workers_refcnt == 0) {
1164                 btrfs_init_workers(&fs_info->scrub_workers, "scrub",
1165                            fs_info->thread_pool_size, &fs_info->generic_worker);
1166                 fs_info->scrub_workers.idle_thresh = 4;
1167                 btrfs_start_workers(&fs_info->scrub_workers, 1);
1168         }
1169         ++fs_info->scrub_workers_refcnt;
1170         mutex_unlock(&fs_info->scrub_lock);
1171
1172         return 0;
1173 }
1174
1175 static noinline_for_stack void scrub_workers_put(struct btrfs_root *root)
1176 {
1177         struct btrfs_fs_info *fs_info = root->fs_info;
1178
1179         mutex_lock(&fs_info->scrub_lock);
1180         if (--fs_info->scrub_workers_refcnt == 0)
1181                 btrfs_stop_workers(&fs_info->scrub_workers);
1182         WARN_ON(fs_info->scrub_workers_refcnt < 0);
1183         mutex_unlock(&fs_info->scrub_lock);
1184 }
1185
1186
1187 int btrfs_scrub_dev(struct btrfs_root *root, u64 devid, u64 start, u64 end,
1188                     struct btrfs_scrub_progress *progress, int readonly)
1189 {
1190         struct scrub_dev *sdev;
1191         struct btrfs_fs_info *fs_info = root->fs_info;
1192         int ret;
1193         struct btrfs_device *dev;
1194
1195         if (btrfs_fs_closing(root->fs_info))
1196                 return -EINVAL;
1197
1198         /*
1199          * check some assumptions
1200          */
1201         if (root->sectorsize != PAGE_SIZE ||
1202             root->sectorsize != root->leafsize ||
1203             root->sectorsize != root->nodesize) {
1204                 printk(KERN_ERR "btrfs_scrub: size assumptions fail\n");
1205                 return -EINVAL;
1206         }
1207
1208         ret = scrub_workers_get(root);
1209         if (ret)
1210                 return ret;
1211
1212         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1213         dev = btrfs_find_device(root, devid, NULL, NULL);
1214         if (!dev || dev->missing) {
1215                 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1216                 scrub_workers_put(root);
1217                 return -ENODEV;
1218         }
1219         mutex_lock(&fs_info->scrub_lock);
1220
1221         if (!dev->in_fs_metadata) {
1222                 mutex_unlock(&fs_info->scrub_lock);
1223                 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1224                 scrub_workers_put(root);
1225                 return -ENODEV;
1226         }
1227
1228         if (dev->scrub_device) {
1229                 mutex_unlock(&fs_info->scrub_lock);
1230                 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1231                 scrub_workers_put(root);
1232                 return -EINPROGRESS;
1233         }
1234         sdev = scrub_setup_dev(dev);
1235         if (IS_ERR(sdev)) {
1236                 mutex_unlock(&fs_info->scrub_lock);
1237                 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1238                 scrub_workers_put(root);
1239                 return PTR_ERR(sdev);
1240         }
1241         sdev->readonly = readonly;
1242         dev->scrub_device = sdev;
1243
1244         atomic_inc(&fs_info->scrubs_running);
1245         mutex_unlock(&fs_info->scrub_lock);
1246         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1247
1248         down_read(&fs_info->scrub_super_lock);
1249         ret = scrub_supers(sdev);
1250         up_read(&fs_info->scrub_super_lock);
1251
1252         if (!ret)
1253                 ret = scrub_enumerate_chunks(sdev, start, end);
1254
1255         wait_event(sdev->list_wait, atomic_read(&sdev->in_flight) == 0);
1256
1257         atomic_dec(&fs_info->scrubs_running);
1258         wake_up(&fs_info->scrub_pause_wait);
1259
1260         if (progress)
1261                 memcpy(progress, &sdev->stat, sizeof(*progress));
1262
1263         mutex_lock(&fs_info->scrub_lock);
1264         dev->scrub_device = NULL;
1265         mutex_unlock(&fs_info->scrub_lock);
1266
1267         scrub_free_dev(sdev);
1268         scrub_workers_put(root);
1269
1270         return ret;
1271 }
1272
1273 int btrfs_scrub_pause(struct btrfs_root *root)
1274 {
1275         struct btrfs_fs_info *fs_info = root->fs_info;
1276
1277         mutex_lock(&fs_info->scrub_lock);
1278         atomic_inc(&fs_info->scrub_pause_req);
1279         while (atomic_read(&fs_info->scrubs_paused) !=
1280                atomic_read(&fs_info->scrubs_running)) {
1281                 mutex_unlock(&fs_info->scrub_lock);
1282                 wait_event(fs_info->scrub_pause_wait,
1283                            atomic_read(&fs_info->scrubs_paused) ==
1284                            atomic_read(&fs_info->scrubs_running));
1285                 mutex_lock(&fs_info->scrub_lock);
1286         }
1287         mutex_unlock(&fs_info->scrub_lock);
1288
1289         return 0;
1290 }
1291
1292 int btrfs_scrub_continue(struct btrfs_root *root)
1293 {
1294         struct btrfs_fs_info *fs_info = root->fs_info;
1295
1296         atomic_dec(&fs_info->scrub_pause_req);
1297         wake_up(&fs_info->scrub_pause_wait);
1298         return 0;
1299 }
1300
1301 int btrfs_scrub_pause_super(struct btrfs_root *root)
1302 {
1303         down_write(&root->fs_info->scrub_super_lock);
1304         return 0;
1305 }
1306
1307 int btrfs_scrub_continue_super(struct btrfs_root *root)
1308 {
1309         up_write(&root->fs_info->scrub_super_lock);
1310         return 0;
1311 }
1312
1313 int btrfs_scrub_cancel(struct btrfs_root *root)
1314 {
1315         struct btrfs_fs_info *fs_info = root->fs_info;
1316
1317         mutex_lock(&fs_info->scrub_lock);
1318         if (!atomic_read(&fs_info->scrubs_running)) {
1319                 mutex_unlock(&fs_info->scrub_lock);
1320                 return -ENOTCONN;
1321         }
1322
1323         atomic_inc(&fs_info->scrub_cancel_req);
1324         while (atomic_read(&fs_info->scrubs_running)) {
1325                 mutex_unlock(&fs_info->scrub_lock);
1326                 wait_event(fs_info->scrub_pause_wait,
1327                            atomic_read(&fs_info->scrubs_running) == 0);
1328                 mutex_lock(&fs_info->scrub_lock);
1329         }
1330         atomic_dec(&fs_info->scrub_cancel_req);
1331         mutex_unlock(&fs_info->scrub_lock);
1332
1333         return 0;
1334 }
1335
1336 int btrfs_scrub_cancel_dev(struct btrfs_root *root, struct btrfs_device *dev)
1337 {
1338         struct btrfs_fs_info *fs_info = root->fs_info;
1339         struct scrub_dev *sdev;
1340
1341         mutex_lock(&fs_info->scrub_lock);
1342         sdev = dev->scrub_device;
1343         if (!sdev) {
1344                 mutex_unlock(&fs_info->scrub_lock);
1345                 return -ENOTCONN;
1346         }
1347         atomic_inc(&sdev->cancel_req);
1348         while (dev->scrub_device) {
1349                 mutex_unlock(&fs_info->scrub_lock);
1350                 wait_event(fs_info->scrub_pause_wait,
1351                            dev->scrub_device == NULL);
1352                 mutex_lock(&fs_info->scrub_lock);
1353         }
1354         mutex_unlock(&fs_info->scrub_lock);
1355
1356         return 0;
1357 }
1358 int btrfs_scrub_cancel_devid(struct btrfs_root *root, u64 devid)
1359 {
1360         struct btrfs_fs_info *fs_info = root->fs_info;
1361         struct btrfs_device *dev;
1362         int ret;
1363
1364         /*
1365          * we have to hold the device_list_mutex here so the device
1366          * does not go away in cancel_dev. FIXME: find a better solution
1367          */
1368         mutex_lock(&fs_info->fs_devices->device_list_mutex);
1369         dev = btrfs_find_device(root, devid, NULL, NULL);
1370         if (!dev) {
1371                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
1372                 return -ENODEV;
1373         }
1374         ret = btrfs_scrub_cancel_dev(root, dev);
1375         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
1376
1377         return ret;
1378 }
1379
1380 int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
1381                          struct btrfs_scrub_progress *progress)
1382 {
1383         struct btrfs_device *dev;
1384         struct scrub_dev *sdev = NULL;
1385
1386         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1387         dev = btrfs_find_device(root, devid, NULL, NULL);
1388         if (dev)
1389                 sdev = dev->scrub_device;
1390         if (sdev)
1391                 memcpy(progress, &sdev->stat, sizeof(*progress));
1392         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1393
1394         return dev ? (sdev ? 0 : -ENOTCONN) : -ENODEV;
1395 }