dmi_scan: fix missing check for _DMI_ signature in smbios_present()
[pandora-kernel.git] / drivers / md / dm-thin.c
1 /*
2  * Copyright (C) 2011 Red Hat UK.
3  *
4  * This file is released under the GPL.
5  */
6
7 #include "dm-thin-metadata.h"
8
9 #include <linux/device-mapper.h>
10 #include <linux/dm-io.h>
11 #include <linux/dm-kcopyd.h>
12 #include <linux/list.h>
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/slab.h>
16
17 #define DM_MSG_PREFIX   "thin"
18
19 /*
20  * Tunable constants
21  */
22 #define ENDIO_HOOK_POOL_SIZE 1024
23 #define DEFERRED_SET_SIZE 64
24 #define MAPPING_POOL_SIZE 1024
25 #define PRISON_CELLS 1024
26
27 /*
28  * The block size of the device holding pool data must be
29  * between 64KB and 1GB.
30  */
31 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
32 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
33
34 /*
35  * The metadata device is currently limited in size.  The limitation is
36  * checked lower down in dm-space-map-metadata, but we also check it here
37  * so we can fail early.
38  *
39  * We have one block of index, which can hold 255 index entries.  Each
40  * index entry contains allocation info about 16k metadata blocks.
41  */
42 #define METADATA_DEV_MAX_SECTORS (255 * (1 << 14) * (THIN_METADATA_BLOCK_SIZE / (1 << SECTOR_SHIFT)))
43
44 /*
45  * Device id is restricted to 24 bits.
46  */
47 #define MAX_DEV_ID ((1 << 24) - 1)
48
49 /*
50  * How do we handle breaking sharing of data blocks?
51  * =================================================
52  *
53  * We use a standard copy-on-write btree to store the mappings for the
54  * devices (note I'm talking about copy-on-write of the metadata here, not
55  * the data).  When you take an internal snapshot you clone the root node
56  * of the origin btree.  After this there is no concept of an origin or a
57  * snapshot.  They are just two device trees that happen to point to the
58  * same data blocks.
59  *
60  * When we get a write in we decide if it's to a shared data block using
61  * some timestamp magic.  If it is, we have to break sharing.
62  *
63  * Let's say we write to a shared block in what was the origin.  The
64  * steps are:
65  *
66  * i) plug io further to this physical block. (see bio_prison code).
67  *
68  * ii) quiesce any read io to that shared data block.  Obviously
69  * including all devices that share this block.  (see deferred_set code)
70  *
71  * iii) copy the data block to a newly allocate block.  This step can be
72  * missed out if the io covers the block. (schedule_copy).
73  *
74  * iv) insert the new mapping into the origin's btree
75  * (process_prepared_mappings).  This act of inserting breaks some
76  * sharing of btree nodes between the two devices.  Breaking sharing only
77  * effects the btree of that specific device.  Btrees for the other
78  * devices that share the block never change.  The btree for the origin
79  * device as it was after the last commit is untouched, ie. we're using
80  * persistent data structures in the functional programming sense.
81  *
82  * v) unplug io to this physical block, including the io that triggered
83  * the breaking of sharing.
84  *
85  * Steps (ii) and (iii) occur in parallel.
86  *
87  * The metadata _doesn't_ need to be committed before the io continues.  We
88  * get away with this because the io is always written to a _new_ block.
89  * If there's a crash, then:
90  *
91  * - The origin mapping will point to the old origin block (the shared
92  * one).  This will contain the data as it was before the io that triggered
93  * the breaking of sharing came in.
94  *
95  * - The snap mapping still points to the old block.  As it would after
96  * the commit.
97  *
98  * The downside of this scheme is the timestamp magic isn't perfect, and
99  * will continue to think that data block in the snapshot device is shared
100  * even after the write to the origin has broken sharing.  I suspect data
101  * blocks will typically be shared by many different devices, so we're
102  * breaking sharing n + 1 times, rather than n, where n is the number of
103  * devices that reference this data block.  At the moment I think the
104  * benefits far, far outweigh the disadvantages.
105  */
106
107 /*----------------------------------------------------------------*/
108
109 /*
110  * Sometimes we can't deal with a bio straight away.  We put them in prison
111  * where they can't cause any mischief.  Bios are put in a cell identified
112  * by a key, multiple bios can be in the same cell.  When the cell is
113  * subsequently unlocked the bios become available.
114  */
115 struct bio_prison;
116
117 struct cell_key {
118         int virtual;
119         dm_thin_id dev;
120         dm_block_t block;
121 };
122
123 struct cell {
124         struct hlist_node list;
125         struct bio_prison *prison;
126         struct cell_key key;
127         struct bio *holder;
128         struct bio_list bios;
129 };
130
131 struct bio_prison {
132         spinlock_t lock;
133         mempool_t *cell_pool;
134
135         unsigned nr_buckets;
136         unsigned hash_mask;
137         struct hlist_head *cells;
138 };
139
140 static uint32_t calc_nr_buckets(unsigned nr_cells)
141 {
142         uint32_t n = 128;
143
144         nr_cells /= 4;
145         nr_cells = min(nr_cells, 8192u);
146
147         while (n < nr_cells)
148                 n <<= 1;
149
150         return n;
151 }
152
153 /*
154  * @nr_cells should be the number of cells you want in use _concurrently_.
155  * Don't confuse it with the number of distinct keys.
156  */
157 static struct bio_prison *prison_create(unsigned nr_cells)
158 {
159         unsigned i;
160         uint32_t nr_buckets = calc_nr_buckets(nr_cells);
161         size_t len = sizeof(struct bio_prison) +
162                 (sizeof(struct hlist_head) * nr_buckets);
163         struct bio_prison *prison = kmalloc(len, GFP_KERNEL);
164
165         if (!prison)
166                 return NULL;
167
168         spin_lock_init(&prison->lock);
169         prison->cell_pool = mempool_create_kmalloc_pool(nr_cells,
170                                                         sizeof(struct cell));
171         if (!prison->cell_pool) {
172                 kfree(prison);
173                 return NULL;
174         }
175
176         prison->nr_buckets = nr_buckets;
177         prison->hash_mask = nr_buckets - 1;
178         prison->cells = (struct hlist_head *) (prison + 1);
179         for (i = 0; i < nr_buckets; i++)
180                 INIT_HLIST_HEAD(prison->cells + i);
181
182         return prison;
183 }
184
185 static void prison_destroy(struct bio_prison *prison)
186 {
187         mempool_destroy(prison->cell_pool);
188         kfree(prison);
189 }
190
191 static uint32_t hash_key(struct bio_prison *prison, struct cell_key *key)
192 {
193         const unsigned long BIG_PRIME = 4294967291UL;
194         uint64_t hash = key->block * BIG_PRIME;
195
196         return (uint32_t) (hash & prison->hash_mask);
197 }
198
199 static int keys_equal(struct cell_key *lhs, struct cell_key *rhs)
200 {
201                return (lhs->virtual == rhs->virtual) &&
202                        (lhs->dev == rhs->dev) &&
203                        (lhs->block == rhs->block);
204 }
205
206 static struct cell *__search_bucket(struct hlist_head *bucket,
207                                     struct cell_key *key)
208 {
209         struct cell *cell;
210         struct hlist_node *tmp;
211
212         hlist_for_each_entry(cell, tmp, bucket, list)
213                 if (keys_equal(&cell->key, key))
214                         return cell;
215
216         return NULL;
217 }
218
219 /*
220  * This may block if a new cell needs allocating.  You must ensure that
221  * cells will be unlocked even if the calling thread is blocked.
222  *
223  * Returns 1 if the cell was already held, 0 if @inmate is the new holder.
224  */
225 static int bio_detain(struct bio_prison *prison, struct cell_key *key,
226                       struct bio *inmate, struct cell **ref)
227 {
228         int r = 1;
229         unsigned long flags;
230         uint32_t hash = hash_key(prison, key);
231         struct cell *cell, *cell2;
232
233         BUG_ON(hash > prison->nr_buckets);
234
235         spin_lock_irqsave(&prison->lock, flags);
236
237         cell = __search_bucket(prison->cells + hash, key);
238         if (cell) {
239                 bio_list_add(&cell->bios, inmate);
240                 goto out;
241         }
242
243         /*
244          * Allocate a new cell
245          */
246         spin_unlock_irqrestore(&prison->lock, flags);
247         cell2 = mempool_alloc(prison->cell_pool, GFP_NOIO);
248         spin_lock_irqsave(&prison->lock, flags);
249
250         /*
251          * We've been unlocked, so we have to double check that
252          * nobody else has inserted this cell in the meantime.
253          */
254         cell = __search_bucket(prison->cells + hash, key);
255         if (cell) {
256                 mempool_free(cell2, prison->cell_pool);
257                 bio_list_add(&cell->bios, inmate);
258                 goto out;
259         }
260
261         /*
262          * Use new cell.
263          */
264         cell = cell2;
265
266         cell->prison = prison;
267         memcpy(&cell->key, key, sizeof(cell->key));
268         cell->holder = inmate;
269         bio_list_init(&cell->bios);
270         hlist_add_head(&cell->list, prison->cells + hash);
271
272         r = 0;
273
274 out:
275         spin_unlock_irqrestore(&prison->lock, flags);
276
277         *ref = cell;
278
279         return r;
280 }
281
282 /*
283  * @inmates must have been initialised prior to this call
284  */
285 static void __cell_release(struct cell *cell, struct bio_list *inmates)
286 {
287         struct bio_prison *prison = cell->prison;
288
289         hlist_del(&cell->list);
290
291         if (inmates) {
292                 bio_list_add(inmates, cell->holder);
293                 bio_list_merge(inmates, &cell->bios);
294         }
295
296         mempool_free(cell, prison->cell_pool);
297 }
298
299 static void cell_release(struct cell *cell, struct bio_list *bios)
300 {
301         unsigned long flags;
302         struct bio_prison *prison = cell->prison;
303
304         spin_lock_irqsave(&prison->lock, flags);
305         __cell_release(cell, bios);
306         spin_unlock_irqrestore(&prison->lock, flags);
307 }
308
309 /*
310  * There are a couple of places where we put a bio into a cell briefly
311  * before taking it out again.  In these situations we know that no other
312  * bio may be in the cell.  This function releases the cell, and also does
313  * a sanity check.
314  */
315 static void __cell_release_singleton(struct cell *cell, struct bio *bio)
316 {
317         BUG_ON(cell->holder != bio);
318         BUG_ON(!bio_list_empty(&cell->bios));
319
320         __cell_release(cell, NULL);
321 }
322
323 static void cell_release_singleton(struct cell *cell, struct bio *bio)
324 {
325         unsigned long flags;
326         struct bio_prison *prison = cell->prison;
327
328         spin_lock_irqsave(&prison->lock, flags);
329         __cell_release_singleton(cell, bio);
330         spin_unlock_irqrestore(&prison->lock, flags);
331 }
332
333 /*
334  * Sometimes we don't want the holder, just the additional bios.
335  */
336 static void __cell_release_no_holder(struct cell *cell, struct bio_list *inmates)
337 {
338         struct bio_prison *prison = cell->prison;
339
340         hlist_del(&cell->list);
341         bio_list_merge(inmates, &cell->bios);
342
343         mempool_free(cell, prison->cell_pool);
344 }
345
346 static void cell_release_no_holder(struct cell *cell, struct bio_list *inmates)
347 {
348         unsigned long flags;
349         struct bio_prison *prison = cell->prison;
350
351         spin_lock_irqsave(&prison->lock, flags);
352         __cell_release_no_holder(cell, inmates);
353         spin_unlock_irqrestore(&prison->lock, flags);
354 }
355
356 static void cell_error(struct cell *cell)
357 {
358         struct bio_prison *prison = cell->prison;
359         struct bio_list bios;
360         struct bio *bio;
361         unsigned long flags;
362
363         bio_list_init(&bios);
364
365         spin_lock_irqsave(&prison->lock, flags);
366         __cell_release(cell, &bios);
367         spin_unlock_irqrestore(&prison->lock, flags);
368
369         while ((bio = bio_list_pop(&bios)))
370                 bio_io_error(bio);
371 }
372
373 /*----------------------------------------------------------------*/
374
375 /*
376  * We use the deferred set to keep track of pending reads to shared blocks.
377  * We do this to ensure the new mapping caused by a write isn't performed
378  * until these prior reads have completed.  Otherwise the insertion of the
379  * new mapping could free the old block that the read bios are mapped to.
380  */
381
382 struct deferred_set;
383 struct deferred_entry {
384         struct deferred_set *ds;
385         unsigned count;
386         struct list_head work_items;
387 };
388
389 struct deferred_set {
390         spinlock_t lock;
391         unsigned current_entry;
392         unsigned sweeper;
393         struct deferred_entry entries[DEFERRED_SET_SIZE];
394 };
395
396 static void ds_init(struct deferred_set *ds)
397 {
398         int i;
399
400         spin_lock_init(&ds->lock);
401         ds->current_entry = 0;
402         ds->sweeper = 0;
403         for (i = 0; i < DEFERRED_SET_SIZE; i++) {
404                 ds->entries[i].ds = ds;
405                 ds->entries[i].count = 0;
406                 INIT_LIST_HEAD(&ds->entries[i].work_items);
407         }
408 }
409
410 static struct deferred_entry *ds_inc(struct deferred_set *ds)
411 {
412         unsigned long flags;
413         struct deferred_entry *entry;
414
415         spin_lock_irqsave(&ds->lock, flags);
416         entry = ds->entries + ds->current_entry;
417         entry->count++;
418         spin_unlock_irqrestore(&ds->lock, flags);
419
420         return entry;
421 }
422
423 static unsigned ds_next(unsigned index)
424 {
425         return (index + 1) % DEFERRED_SET_SIZE;
426 }
427
428 static void __sweep(struct deferred_set *ds, struct list_head *head)
429 {
430         while ((ds->sweeper != ds->current_entry) &&
431                !ds->entries[ds->sweeper].count) {
432                 list_splice_init(&ds->entries[ds->sweeper].work_items, head);
433                 ds->sweeper = ds_next(ds->sweeper);
434         }
435
436         if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
437                 list_splice_init(&ds->entries[ds->sweeper].work_items, head);
438 }
439
440 static void ds_dec(struct deferred_entry *entry, struct list_head *head)
441 {
442         unsigned long flags;
443
444         spin_lock_irqsave(&entry->ds->lock, flags);
445         BUG_ON(!entry->count);
446         --entry->count;
447         __sweep(entry->ds, head);
448         spin_unlock_irqrestore(&entry->ds->lock, flags);
449 }
450
451 /*
452  * Returns 1 if deferred or 0 if no pending items to delay job.
453  */
454 static int ds_add_work(struct deferred_set *ds, struct list_head *work)
455 {
456         int r = 1;
457         unsigned long flags;
458         unsigned next_entry;
459
460         spin_lock_irqsave(&ds->lock, flags);
461         if ((ds->sweeper == ds->current_entry) &&
462             !ds->entries[ds->current_entry].count)
463                 r = 0;
464         else {
465                 list_add(work, &ds->entries[ds->current_entry].work_items);
466                 next_entry = ds_next(ds->current_entry);
467                 if (!ds->entries[next_entry].count)
468                         ds->current_entry = next_entry;
469         }
470         spin_unlock_irqrestore(&ds->lock, flags);
471
472         return r;
473 }
474
475 /*----------------------------------------------------------------*/
476
477 /*
478  * Key building.
479  */
480 static void build_data_key(struct dm_thin_device *td,
481                            dm_block_t b, struct cell_key *key)
482 {
483         key->virtual = 0;
484         key->dev = dm_thin_dev_id(td);
485         key->block = b;
486 }
487
488 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
489                               struct cell_key *key)
490 {
491         key->virtual = 1;
492         key->dev = dm_thin_dev_id(td);
493         key->block = b;
494 }
495
496 /*----------------------------------------------------------------*/
497
498 /*
499  * A pool device ties together a metadata device and a data device.  It
500  * also provides the interface for creating and destroying internal
501  * devices.
502  */
503 struct new_mapping;
504 struct pool {
505         struct list_head list;
506         struct dm_target *ti;   /* Only set if a pool target is bound */
507
508         struct mapped_device *pool_md;
509         struct block_device *md_dev;
510         struct dm_pool_metadata *pmd;
511
512         uint32_t sectors_per_block;
513         unsigned block_shift;
514         dm_block_t offset_mask;
515         dm_block_t low_water_blocks;
516
517         unsigned zero_new_blocks:1;
518         unsigned low_water_triggered:1; /* A dm event has been sent */
519         unsigned no_free_space:1;       /* A -ENOSPC warning has been issued */
520
521         struct bio_prison *prison;
522         struct dm_kcopyd_client *copier;
523
524         struct workqueue_struct *wq;
525         struct work_struct worker;
526
527         unsigned ref_count;
528
529         spinlock_t lock;
530         struct bio_list deferred_bios;
531         struct bio_list deferred_flush_bios;
532         struct list_head prepared_mappings;
533
534         struct bio_list retry_on_resume_list;
535
536         struct deferred_set ds; /* FIXME: move to thin_c */
537
538         struct new_mapping *next_mapping;
539         mempool_t *mapping_pool;
540         mempool_t *endio_hook_pool;
541 };
542
543 /*
544  * Target context for a pool.
545  */
546 struct pool_c {
547         struct dm_target *ti;
548         struct pool *pool;
549         struct dm_dev *data_dev;
550         struct dm_dev *metadata_dev;
551         struct dm_target_callbacks callbacks;
552
553         dm_block_t low_water_blocks;
554         unsigned zero_new_blocks:1;
555 };
556
557 /*
558  * Target context for a thin.
559  */
560 struct thin_c {
561         struct dm_dev *pool_dev;
562         dm_thin_id dev_id;
563
564         struct pool *pool;
565         struct dm_thin_device *td;
566 };
567
568 /*----------------------------------------------------------------*/
569
570 /*
571  * A global list of pools that uses a struct mapped_device as a key.
572  */
573 static struct dm_thin_pool_table {
574         struct mutex mutex;
575         struct list_head pools;
576 } dm_thin_pool_table;
577
578 static void pool_table_init(void)
579 {
580         mutex_init(&dm_thin_pool_table.mutex);
581         INIT_LIST_HEAD(&dm_thin_pool_table.pools);
582 }
583
584 static void __pool_table_insert(struct pool *pool)
585 {
586         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
587         list_add(&pool->list, &dm_thin_pool_table.pools);
588 }
589
590 static void __pool_table_remove(struct pool *pool)
591 {
592         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
593         list_del(&pool->list);
594 }
595
596 static struct pool *__pool_table_lookup(struct mapped_device *md)
597 {
598         struct pool *pool = NULL, *tmp;
599
600         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
601
602         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
603                 if (tmp->pool_md == md) {
604                         pool = tmp;
605                         break;
606                 }
607         }
608
609         return pool;
610 }
611
612 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
613 {
614         struct pool *pool = NULL, *tmp;
615
616         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
617
618         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
619                 if (tmp->md_dev == md_dev) {
620                         pool = tmp;
621                         break;
622                 }
623         }
624
625         return pool;
626 }
627
628 /*----------------------------------------------------------------*/
629
630 static void __requeue_bio_list(struct thin_c *tc, struct bio_list *master)
631 {
632         struct bio *bio;
633         struct bio_list bios;
634
635         bio_list_init(&bios);
636         bio_list_merge(&bios, master);
637         bio_list_init(master);
638
639         while ((bio = bio_list_pop(&bios))) {
640                 if (dm_get_mapinfo(bio)->ptr == tc)
641                         bio_endio(bio, DM_ENDIO_REQUEUE);
642                 else
643                         bio_list_add(master, bio);
644         }
645 }
646
647 static void requeue_io(struct thin_c *tc)
648 {
649         struct pool *pool = tc->pool;
650         unsigned long flags;
651
652         spin_lock_irqsave(&pool->lock, flags);
653         __requeue_bio_list(tc, &pool->deferred_bios);
654         __requeue_bio_list(tc, &pool->retry_on_resume_list);
655         spin_unlock_irqrestore(&pool->lock, flags);
656 }
657
658 /*
659  * This section of code contains the logic for processing a thin device's IO.
660  * Much of the code depends on pool object resources (lists, workqueues, etc)
661  * but most is exclusively called from the thin target rather than the thin-pool
662  * target.
663  */
664
665 static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
666 {
667         return bio->bi_sector >> tc->pool->block_shift;
668 }
669
670 static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
671 {
672         struct pool *pool = tc->pool;
673
674         bio->bi_bdev = tc->pool_dev->bdev;
675         bio->bi_sector = (block << pool->block_shift) +
676                 (bio->bi_sector & pool->offset_mask);
677 }
678
679 static void remap_and_issue(struct thin_c *tc, struct bio *bio,
680                             dm_block_t block)
681 {
682         struct pool *pool = tc->pool;
683         unsigned long flags;
684
685         remap(tc, bio, block);
686
687         /*
688          * Batch together any FUA/FLUSH bios we find and then issue
689          * a single commit for them in process_deferred_bios().
690          */
691         if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
692                 spin_lock_irqsave(&pool->lock, flags);
693                 bio_list_add(&pool->deferred_flush_bios, bio);
694                 spin_unlock_irqrestore(&pool->lock, flags);
695         } else
696                 generic_make_request(bio);
697 }
698
699 /*
700  * wake_worker() is used when new work is queued and when pool_resume is
701  * ready to continue deferred IO processing.
702  */
703 static void wake_worker(struct pool *pool)
704 {
705         queue_work(pool->wq, &pool->worker);
706 }
707
708 /*----------------------------------------------------------------*/
709
710 /*
711  * Bio endio functions.
712  */
713 struct endio_hook {
714         struct thin_c *tc;
715         bio_end_io_t *saved_bi_end_io;
716         struct deferred_entry *entry;
717 };
718
719 struct new_mapping {
720         struct list_head list;
721
722         int prepared;
723
724         struct thin_c *tc;
725         dm_block_t virt_block;
726         dm_block_t data_block;
727         struct cell *cell;
728         int err;
729
730         /*
731          * If the bio covers the whole area of a block then we can avoid
732          * zeroing or copying.  Instead this bio is hooked.  The bio will
733          * still be in the cell, so care has to be taken to avoid issuing
734          * the bio twice.
735          */
736         struct bio *bio;
737         bio_end_io_t *saved_bi_end_io;
738 };
739
740 static void __maybe_add_mapping(struct new_mapping *m)
741 {
742         struct pool *pool = m->tc->pool;
743
744         if (list_empty(&m->list) && m->prepared) {
745                 list_add(&m->list, &pool->prepared_mappings);
746                 wake_worker(pool);
747         }
748 }
749
750 static void copy_complete(int read_err, unsigned long write_err, void *context)
751 {
752         unsigned long flags;
753         struct new_mapping *m = context;
754         struct pool *pool = m->tc->pool;
755
756         m->err = read_err || write_err ? -EIO : 0;
757
758         spin_lock_irqsave(&pool->lock, flags);
759         m->prepared = 1;
760         __maybe_add_mapping(m);
761         spin_unlock_irqrestore(&pool->lock, flags);
762 }
763
764 static void overwrite_endio(struct bio *bio, int err)
765 {
766         unsigned long flags;
767         struct new_mapping *m = dm_get_mapinfo(bio)->ptr;
768         struct pool *pool = m->tc->pool;
769
770         m->err = err;
771
772         spin_lock_irqsave(&pool->lock, flags);
773         m->prepared = 1;
774         __maybe_add_mapping(m);
775         spin_unlock_irqrestore(&pool->lock, flags);
776 }
777
778 static void shared_read_endio(struct bio *bio, int err)
779 {
780         struct list_head mappings;
781         struct new_mapping *m, *tmp;
782         struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
783         unsigned long flags;
784         struct pool *pool = h->tc->pool;
785
786         bio->bi_end_io = h->saved_bi_end_io;
787         bio_endio(bio, err);
788
789         INIT_LIST_HEAD(&mappings);
790         ds_dec(h->entry, &mappings);
791
792         spin_lock_irqsave(&pool->lock, flags);
793         list_for_each_entry_safe(m, tmp, &mappings, list) {
794                 list_del(&m->list);
795                 INIT_LIST_HEAD(&m->list);
796                 __maybe_add_mapping(m);
797         }
798         spin_unlock_irqrestore(&pool->lock, flags);
799
800         mempool_free(h, pool->endio_hook_pool);
801 }
802
803 /*----------------------------------------------------------------*/
804
805 /*
806  * Workqueue.
807  */
808
809 /*
810  * Prepared mapping jobs.
811  */
812
813 /*
814  * This sends the bios in the cell back to the deferred_bios list.
815  */
816 static void cell_defer(struct thin_c *tc, struct cell *cell,
817                        dm_block_t data_block)
818 {
819         struct pool *pool = tc->pool;
820         unsigned long flags;
821
822         spin_lock_irqsave(&pool->lock, flags);
823         cell_release(cell, &pool->deferred_bios);
824         spin_unlock_irqrestore(&tc->pool->lock, flags);
825
826         wake_worker(pool);
827 }
828
829 /*
830  * Same as cell_defer above, except it omits one particular detainee,
831  * a write bio that covers the block and has already been processed.
832  */
833 static void cell_defer_except(struct thin_c *tc, struct cell *cell)
834 {
835         struct bio_list bios;
836         struct pool *pool = tc->pool;
837         unsigned long flags;
838
839         bio_list_init(&bios);
840
841         spin_lock_irqsave(&pool->lock, flags);
842         cell_release_no_holder(cell, &pool->deferred_bios);
843         spin_unlock_irqrestore(&pool->lock, flags);
844
845         wake_worker(pool);
846 }
847
848 static void process_prepared_mapping(struct new_mapping *m)
849 {
850         struct thin_c *tc = m->tc;
851         struct bio *bio;
852         int r;
853
854         bio = m->bio;
855         if (bio)
856                 bio->bi_end_io = m->saved_bi_end_io;
857
858         if (m->err) {
859                 cell_error(m->cell);
860                 goto out;
861         }
862
863         /*
864          * Commit the prepared block into the mapping btree.
865          * Any I/O for this block arriving after this point will get
866          * remapped to it directly.
867          */
868         r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block);
869         if (r) {
870                 DMERR("dm_thin_insert_block() failed");
871                 cell_error(m->cell);
872                 goto out;
873         }
874
875         /*
876          * Release any bios held while the block was being provisioned.
877          * If we are processing a write bio that completely covers the block,
878          * we already processed it so can ignore it now when processing
879          * the bios in the cell.
880          */
881         if (bio) {
882                 cell_defer_except(tc, m->cell);
883                 bio_endio(bio, 0);
884         } else
885                 cell_defer(tc, m->cell, m->data_block);
886
887 out:
888         list_del(&m->list);
889         mempool_free(m, tc->pool->mapping_pool);
890 }
891
892 static void process_prepared_mappings(struct pool *pool)
893 {
894         unsigned long flags;
895         struct list_head maps;
896         struct new_mapping *m, *tmp;
897
898         INIT_LIST_HEAD(&maps);
899         spin_lock_irqsave(&pool->lock, flags);
900         list_splice_init(&pool->prepared_mappings, &maps);
901         spin_unlock_irqrestore(&pool->lock, flags);
902
903         list_for_each_entry_safe(m, tmp, &maps, list)
904                 process_prepared_mapping(m);
905 }
906
907 /*
908  * Deferred bio jobs.
909  */
910 static int io_overwrites_block(struct pool *pool, struct bio *bio)
911 {
912         return ((bio_data_dir(bio) == WRITE) &&
913                 !(bio->bi_sector & pool->offset_mask)) &&
914                 (bio->bi_size == (pool->sectors_per_block << SECTOR_SHIFT));
915 }
916
917 static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
918                                bio_end_io_t *fn)
919 {
920         *save = bio->bi_end_io;
921         bio->bi_end_io = fn;
922 }
923
924 static int ensure_next_mapping(struct pool *pool)
925 {
926         if (pool->next_mapping)
927                 return 0;
928
929         pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
930
931         return pool->next_mapping ? 0 : -ENOMEM;
932 }
933
934 static struct new_mapping *get_next_mapping(struct pool *pool)
935 {
936         struct new_mapping *r = pool->next_mapping;
937
938         BUG_ON(!pool->next_mapping);
939
940         pool->next_mapping = NULL;
941
942         return r;
943 }
944
945 static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
946                           dm_block_t data_origin, dm_block_t data_dest,
947                           struct cell *cell, struct bio *bio)
948 {
949         int r;
950         struct pool *pool = tc->pool;
951         struct new_mapping *m = get_next_mapping(pool);
952
953         INIT_LIST_HEAD(&m->list);
954         m->prepared = 0;
955         m->tc = tc;
956         m->virt_block = virt_block;
957         m->data_block = data_dest;
958         m->cell = cell;
959         m->err = 0;
960         m->bio = NULL;
961
962         ds_add_work(&pool->ds, &m->list);
963
964         /*
965          * IO to pool_dev remaps to the pool target's data_dev.
966          *
967          * If the whole block of data is being overwritten, we can issue the
968          * bio immediately. Otherwise we use kcopyd to clone the data first.
969          */
970         if (io_overwrites_block(pool, bio)) {
971                 m->bio = bio;
972                 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
973                 dm_get_mapinfo(bio)->ptr = m;
974                 remap_and_issue(tc, bio, data_dest);
975         } else {
976                 struct dm_io_region from, to;
977
978                 from.bdev = tc->pool_dev->bdev;
979                 from.sector = data_origin * pool->sectors_per_block;
980                 from.count = pool->sectors_per_block;
981
982                 to.bdev = tc->pool_dev->bdev;
983                 to.sector = data_dest * pool->sectors_per_block;
984                 to.count = pool->sectors_per_block;
985
986                 r = dm_kcopyd_copy(pool->copier, &from, 1, &to,
987                                    0, copy_complete, m);
988                 if (r < 0) {
989                         mempool_free(m, pool->mapping_pool);
990                         DMERR("dm_kcopyd_copy() failed");
991                         cell_error(cell);
992                 }
993         }
994 }
995
996 static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
997                           dm_block_t data_block, struct cell *cell,
998                           struct bio *bio)
999 {
1000         struct pool *pool = tc->pool;
1001         struct new_mapping *m = get_next_mapping(pool);
1002
1003         INIT_LIST_HEAD(&m->list);
1004         m->prepared = 0;
1005         m->tc = tc;
1006         m->virt_block = virt_block;
1007         m->data_block = data_block;
1008         m->cell = cell;
1009         m->err = 0;
1010         m->bio = NULL;
1011
1012         /*
1013          * If the whole block of data is being overwritten or we are not
1014          * zeroing pre-existing data, we can issue the bio immediately.
1015          * Otherwise we use kcopyd to zero the data first.
1016          */
1017         if (!pool->zero_new_blocks)
1018                 process_prepared_mapping(m);
1019
1020         else if (io_overwrites_block(pool, bio)) {
1021                 m->bio = bio;
1022                 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
1023                 dm_get_mapinfo(bio)->ptr = m;
1024                 remap_and_issue(tc, bio, data_block);
1025
1026         } else {
1027                 int r;
1028                 struct dm_io_region to;
1029
1030                 to.bdev = tc->pool_dev->bdev;
1031                 to.sector = data_block * pool->sectors_per_block;
1032                 to.count = pool->sectors_per_block;
1033
1034                 r = dm_kcopyd_zero(pool->copier, 1, &to, 0, copy_complete, m);
1035                 if (r < 0) {
1036                         mempool_free(m, pool->mapping_pool);
1037                         DMERR("dm_kcopyd_zero() failed");
1038                         cell_error(cell);
1039                 }
1040         }
1041 }
1042
1043 static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1044 {
1045         int r;
1046         dm_block_t free_blocks;
1047         unsigned long flags;
1048         struct pool *pool = tc->pool;
1049
1050         r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1051         if (r)
1052                 return r;
1053
1054         if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1055                 DMWARN("%s: reached low water mark, sending event.",
1056                        dm_device_name(pool->pool_md));
1057                 spin_lock_irqsave(&pool->lock, flags);
1058                 pool->low_water_triggered = 1;
1059                 spin_unlock_irqrestore(&pool->lock, flags);
1060                 dm_table_event(pool->ti->table);
1061         }
1062
1063         if (!free_blocks) {
1064                 if (pool->no_free_space)
1065                         return -ENOSPC;
1066                 else {
1067                         /*
1068                          * Try to commit to see if that will free up some
1069                          * more space.
1070                          */
1071                         r = dm_pool_commit_metadata(pool->pmd);
1072                         if (r) {
1073                                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1074                                       __func__, r);
1075                                 return r;
1076                         }
1077
1078                         r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1079                         if (r)
1080                                 return r;
1081
1082                         /*
1083                          * If we still have no space we set a flag to avoid
1084                          * doing all this checking and return -ENOSPC.
1085                          */
1086                         if (!free_blocks) {
1087                                 DMWARN("%s: no free space available.",
1088                                        dm_device_name(pool->pool_md));
1089                                 spin_lock_irqsave(&pool->lock, flags);
1090                                 pool->no_free_space = 1;
1091                                 spin_unlock_irqrestore(&pool->lock, flags);
1092                                 return -ENOSPC;
1093                         }
1094                 }
1095         }
1096
1097         r = dm_pool_alloc_data_block(pool->pmd, result);
1098         if (r)
1099                 return r;
1100
1101         return 0;
1102 }
1103
1104 /*
1105  * If we have run out of space, queue bios until the device is
1106  * resumed, presumably after having been reloaded with more space.
1107  */
1108 static void retry_on_resume(struct bio *bio)
1109 {
1110         struct thin_c *tc = dm_get_mapinfo(bio)->ptr;
1111         struct pool *pool = tc->pool;
1112         unsigned long flags;
1113
1114         spin_lock_irqsave(&pool->lock, flags);
1115         bio_list_add(&pool->retry_on_resume_list, bio);
1116         spin_unlock_irqrestore(&pool->lock, flags);
1117 }
1118
1119 static void no_space(struct cell *cell)
1120 {
1121         struct bio *bio;
1122         struct bio_list bios;
1123
1124         bio_list_init(&bios);
1125         cell_release(cell, &bios);
1126
1127         while ((bio = bio_list_pop(&bios)))
1128                 retry_on_resume(bio);
1129 }
1130
1131 static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
1132                           struct cell_key *key,
1133                           struct dm_thin_lookup_result *lookup_result,
1134                           struct cell *cell)
1135 {
1136         int r;
1137         dm_block_t data_block;
1138
1139         r = alloc_data_block(tc, &data_block);
1140         switch (r) {
1141         case 0:
1142                 schedule_copy(tc, block, lookup_result->block,
1143                               data_block, cell, bio);
1144                 break;
1145
1146         case -ENOSPC:
1147                 no_space(cell);
1148                 break;
1149
1150         default:
1151                 DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);
1152                 cell_error(cell);
1153                 break;
1154         }
1155 }
1156
1157 static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1158                                dm_block_t block,
1159                                struct dm_thin_lookup_result *lookup_result)
1160 {
1161         struct cell *cell;
1162         struct pool *pool = tc->pool;
1163         struct cell_key key;
1164
1165         /*
1166          * If cell is already occupied, then sharing is already in the process
1167          * of being broken so we have nothing further to do here.
1168          */
1169         build_data_key(tc->td, lookup_result->block, &key);
1170         if (bio_detain(pool->prison, &key, bio, &cell))
1171                 return;
1172
1173         if (bio_data_dir(bio) == WRITE)
1174                 break_sharing(tc, bio, block, &key, lookup_result, cell);
1175         else {
1176                 struct endio_hook *h;
1177                 h = mempool_alloc(pool->endio_hook_pool, GFP_NOIO);
1178
1179                 h->tc = tc;
1180                 h->entry = ds_inc(&pool->ds);
1181                 save_and_set_endio(bio, &h->saved_bi_end_io, shared_read_endio);
1182                 dm_get_mapinfo(bio)->ptr = h;
1183
1184                 cell_release_singleton(cell, bio);
1185                 remap_and_issue(tc, bio, lookup_result->block);
1186         }
1187 }
1188
1189 static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
1190                             struct cell *cell)
1191 {
1192         int r;
1193         dm_block_t data_block;
1194
1195         /*
1196          * Remap empty bios (flushes) immediately, without provisioning.
1197          */
1198         if (!bio->bi_size) {
1199                 cell_release_singleton(cell, bio);
1200                 remap_and_issue(tc, bio, 0);
1201                 return;
1202         }
1203
1204         /*
1205          * Fill read bios with zeroes and complete them immediately.
1206          */
1207         if (bio_data_dir(bio) == READ) {
1208                 zero_fill_bio(bio);
1209                 cell_release_singleton(cell, bio);
1210                 bio_endio(bio, 0);
1211                 return;
1212         }
1213
1214         r = alloc_data_block(tc, &data_block);
1215         switch (r) {
1216         case 0:
1217                 schedule_zero(tc, block, data_block, cell, bio);
1218                 break;
1219
1220         case -ENOSPC:
1221                 no_space(cell);
1222                 break;
1223
1224         default:
1225                 DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);
1226                 cell_error(cell);
1227                 break;
1228         }
1229 }
1230
1231 static void process_bio(struct thin_c *tc, struct bio *bio)
1232 {
1233         int r;
1234         dm_block_t block = get_bio_block(tc, bio);
1235         struct cell *cell;
1236         struct cell_key key;
1237         struct dm_thin_lookup_result lookup_result;
1238
1239         /*
1240          * If cell is already occupied, then the block is already
1241          * being provisioned so we have nothing further to do here.
1242          */
1243         build_virtual_key(tc->td, block, &key);
1244         if (bio_detain(tc->pool->prison, &key, bio, &cell))
1245                 return;
1246
1247         r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1248         switch (r) {
1249         case 0:
1250                 /*
1251                  * We can release this cell now.  This thread is the only
1252                  * one that puts bios into a cell, and we know there were
1253                  * no preceding bios.
1254                  */
1255                 /*
1256                  * TODO: this will probably have to change when discard goes
1257                  * back in.
1258                  */
1259                 cell_release_singleton(cell, bio);
1260
1261                 if (lookup_result.shared)
1262                         process_shared_bio(tc, bio, block, &lookup_result);
1263                 else
1264                         remap_and_issue(tc, bio, lookup_result.block);
1265                 break;
1266
1267         case -ENODATA:
1268                 provision_block(tc, bio, block, cell);
1269                 break;
1270
1271         default:
1272                 DMERR("dm_thin_find_block() failed, error = %d", r);
1273                 bio_io_error(bio);
1274                 break;
1275         }
1276 }
1277
1278 static void process_deferred_bios(struct pool *pool)
1279 {
1280         unsigned long flags;
1281         struct bio *bio;
1282         struct bio_list bios;
1283         int r;
1284
1285         bio_list_init(&bios);
1286
1287         spin_lock_irqsave(&pool->lock, flags);
1288         bio_list_merge(&bios, &pool->deferred_bios);
1289         bio_list_init(&pool->deferred_bios);
1290         spin_unlock_irqrestore(&pool->lock, flags);
1291
1292         while ((bio = bio_list_pop(&bios))) {
1293                 struct thin_c *tc = dm_get_mapinfo(bio)->ptr;
1294                 /*
1295                  * If we've got no free new_mapping structs, and processing
1296                  * this bio might require one, we pause until there are some
1297                  * prepared mappings to process.
1298                  */
1299                 if (ensure_next_mapping(pool)) {
1300                         spin_lock_irqsave(&pool->lock, flags);
1301                         bio_list_merge(&pool->deferred_bios, &bios);
1302                         spin_unlock_irqrestore(&pool->lock, flags);
1303
1304                         break;
1305                 }
1306                 process_bio(tc, bio);
1307         }
1308
1309         /*
1310          * If there are any deferred flush bios, we must commit
1311          * the metadata before issuing them.
1312          */
1313         bio_list_init(&bios);
1314         spin_lock_irqsave(&pool->lock, flags);
1315         bio_list_merge(&bios, &pool->deferred_flush_bios);
1316         bio_list_init(&pool->deferred_flush_bios);
1317         spin_unlock_irqrestore(&pool->lock, flags);
1318
1319         if (bio_list_empty(&bios))
1320                 return;
1321
1322         r = dm_pool_commit_metadata(pool->pmd);
1323         if (r) {
1324                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1325                       __func__, r);
1326                 while ((bio = bio_list_pop(&bios)))
1327                         bio_io_error(bio);
1328                 return;
1329         }
1330
1331         while ((bio = bio_list_pop(&bios)))
1332                 generic_make_request(bio);
1333 }
1334
1335 static void do_worker(struct work_struct *ws)
1336 {
1337         struct pool *pool = container_of(ws, struct pool, worker);
1338
1339         process_prepared_mappings(pool);
1340         process_deferred_bios(pool);
1341 }
1342
1343 /*----------------------------------------------------------------*/
1344
1345 /*
1346  * Mapping functions.
1347  */
1348
1349 /*
1350  * Called only while mapping a thin bio to hand it over to the workqueue.
1351  */
1352 static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
1353 {
1354         unsigned long flags;
1355         struct pool *pool = tc->pool;
1356
1357         spin_lock_irqsave(&pool->lock, flags);
1358         bio_list_add(&pool->deferred_bios, bio);
1359         spin_unlock_irqrestore(&pool->lock, flags);
1360
1361         wake_worker(pool);
1362 }
1363
1364 /*
1365  * Non-blocking function called from the thin target's map function.
1366  */
1367 static int thin_bio_map(struct dm_target *ti, struct bio *bio,
1368                         union map_info *map_context)
1369 {
1370         int r;
1371         struct thin_c *tc = ti->private;
1372         dm_block_t block = get_bio_block(tc, bio);
1373         struct dm_thin_device *td = tc->td;
1374         struct dm_thin_lookup_result result;
1375
1376         /*
1377          * Save the thin context for easy access from the deferred bio later.
1378          */
1379         map_context->ptr = tc;
1380
1381         if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
1382                 thin_defer_bio(tc, bio);
1383                 return DM_MAPIO_SUBMITTED;
1384         }
1385
1386         r = dm_thin_find_block(td, block, 0, &result);
1387
1388         /*
1389          * Note that we defer readahead too.
1390          */
1391         switch (r) {
1392         case 0:
1393                 if (unlikely(result.shared)) {
1394                         /*
1395                          * We have a race condition here between the
1396                          * result.shared value returned by the lookup and
1397                          * snapshot creation, which may cause new
1398                          * sharing.
1399                          *
1400                          * To avoid this always quiesce the origin before
1401                          * taking the snap.  You want to do this anyway to
1402                          * ensure a consistent application view
1403                          * (i.e. lockfs).
1404                          *
1405                          * More distant ancestors are irrelevant. The
1406                          * shared flag will be set in their case.
1407                          */
1408                         thin_defer_bio(tc, bio);
1409                         r = DM_MAPIO_SUBMITTED;
1410                 } else {
1411                         remap(tc, bio, result.block);
1412                         r = DM_MAPIO_REMAPPED;
1413                 }
1414                 break;
1415
1416         case -ENODATA:
1417                 /*
1418                  * In future, the failed dm_thin_find_block above could
1419                  * provide the hint to load the metadata into cache.
1420                  */
1421         case -EWOULDBLOCK:
1422                 thin_defer_bio(tc, bio);
1423                 r = DM_MAPIO_SUBMITTED;
1424                 break;
1425         }
1426
1427         return r;
1428 }
1429
1430 static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
1431 {
1432         int r;
1433         unsigned long flags;
1434         struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
1435
1436         spin_lock_irqsave(&pt->pool->lock, flags);
1437         r = !bio_list_empty(&pt->pool->retry_on_resume_list);
1438         spin_unlock_irqrestore(&pt->pool->lock, flags);
1439
1440         if (!r) {
1441                 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
1442                 r = bdi_congested(&q->backing_dev_info, bdi_bits);
1443         }
1444
1445         return r;
1446 }
1447
1448 static void __requeue_bios(struct pool *pool)
1449 {
1450         bio_list_merge(&pool->deferred_bios, &pool->retry_on_resume_list);
1451         bio_list_init(&pool->retry_on_resume_list);
1452 }
1453
1454 /*----------------------------------------------------------------
1455  * Binding of control targets to a pool object
1456  *--------------------------------------------------------------*/
1457 static int bind_control_target(struct pool *pool, struct dm_target *ti)
1458 {
1459         struct pool_c *pt = ti->private;
1460
1461         pool->ti = ti;
1462         pool->low_water_blocks = pt->low_water_blocks;
1463         pool->zero_new_blocks = pt->zero_new_blocks;
1464
1465         return 0;
1466 }
1467
1468 static void unbind_control_target(struct pool *pool, struct dm_target *ti)
1469 {
1470         if (pool->ti == ti)
1471                 pool->ti = NULL;
1472 }
1473
1474 /*----------------------------------------------------------------
1475  * Pool creation
1476  *--------------------------------------------------------------*/
1477 static void __pool_destroy(struct pool *pool)
1478 {
1479         __pool_table_remove(pool);
1480
1481         if (dm_pool_metadata_close(pool->pmd) < 0)
1482                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
1483
1484         prison_destroy(pool->prison);
1485         dm_kcopyd_client_destroy(pool->copier);
1486
1487         if (pool->wq)
1488                 destroy_workqueue(pool->wq);
1489
1490         if (pool->next_mapping)
1491                 mempool_free(pool->next_mapping, pool->mapping_pool);
1492         mempool_destroy(pool->mapping_pool);
1493         mempool_destroy(pool->endio_hook_pool);
1494         kfree(pool);
1495 }
1496
1497 static struct pool *pool_create(struct mapped_device *pool_md,
1498                                 struct block_device *metadata_dev,
1499                                 unsigned long block_size, char **error)
1500 {
1501         int r;
1502         void *err_p;
1503         struct pool *pool;
1504         struct dm_pool_metadata *pmd;
1505
1506         pmd = dm_pool_metadata_open(metadata_dev, block_size);
1507         if (IS_ERR(pmd)) {
1508                 *error = "Error creating metadata object";
1509                 return (struct pool *)pmd;
1510         }
1511
1512         pool = kmalloc(sizeof(*pool), GFP_KERNEL);
1513         if (!pool) {
1514                 *error = "Error allocating memory for pool";
1515                 err_p = ERR_PTR(-ENOMEM);
1516                 goto bad_pool;
1517         }
1518
1519         pool->pmd = pmd;
1520         pool->sectors_per_block = block_size;
1521         pool->block_shift = ffs(block_size) - 1;
1522         pool->offset_mask = block_size - 1;
1523         pool->low_water_blocks = 0;
1524         pool->zero_new_blocks = 1;
1525         pool->prison = prison_create(PRISON_CELLS);
1526         if (!pool->prison) {
1527                 *error = "Error creating pool's bio prison";
1528                 err_p = ERR_PTR(-ENOMEM);
1529                 goto bad_prison;
1530         }
1531
1532         pool->copier = dm_kcopyd_client_create();
1533         if (IS_ERR(pool->copier)) {
1534                 r = PTR_ERR(pool->copier);
1535                 *error = "Error creating pool's kcopyd client";
1536                 err_p = ERR_PTR(r);
1537                 goto bad_kcopyd_client;
1538         }
1539
1540         /*
1541          * Create singlethreaded workqueue that will service all devices
1542          * that use this metadata.
1543          */
1544         pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
1545         if (!pool->wq) {
1546                 *error = "Error creating pool's workqueue";
1547                 err_p = ERR_PTR(-ENOMEM);
1548                 goto bad_wq;
1549         }
1550
1551         INIT_WORK(&pool->worker, do_worker);
1552         spin_lock_init(&pool->lock);
1553         bio_list_init(&pool->deferred_bios);
1554         bio_list_init(&pool->deferred_flush_bios);
1555         INIT_LIST_HEAD(&pool->prepared_mappings);
1556         pool->low_water_triggered = 0;
1557         pool->no_free_space = 0;
1558         bio_list_init(&pool->retry_on_resume_list);
1559         ds_init(&pool->ds);
1560
1561         pool->next_mapping = NULL;
1562         pool->mapping_pool =
1563                 mempool_create_kmalloc_pool(MAPPING_POOL_SIZE, sizeof(struct new_mapping));
1564         if (!pool->mapping_pool) {
1565                 *error = "Error creating pool's mapping mempool";
1566                 err_p = ERR_PTR(-ENOMEM);
1567                 goto bad_mapping_pool;
1568         }
1569
1570         pool->endio_hook_pool =
1571                 mempool_create_kmalloc_pool(ENDIO_HOOK_POOL_SIZE, sizeof(struct endio_hook));
1572         if (!pool->endio_hook_pool) {
1573                 *error = "Error creating pool's endio_hook mempool";
1574                 err_p = ERR_PTR(-ENOMEM);
1575                 goto bad_endio_hook_pool;
1576         }
1577         pool->ref_count = 1;
1578         pool->pool_md = pool_md;
1579         pool->md_dev = metadata_dev;
1580         __pool_table_insert(pool);
1581
1582         return pool;
1583
1584 bad_endio_hook_pool:
1585         mempool_destroy(pool->mapping_pool);
1586 bad_mapping_pool:
1587         destroy_workqueue(pool->wq);
1588 bad_wq:
1589         dm_kcopyd_client_destroy(pool->copier);
1590 bad_kcopyd_client:
1591         prison_destroy(pool->prison);
1592 bad_prison:
1593         kfree(pool);
1594 bad_pool:
1595         if (dm_pool_metadata_close(pmd))
1596                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
1597
1598         return err_p;
1599 }
1600
1601 static void __pool_inc(struct pool *pool)
1602 {
1603         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
1604         pool->ref_count++;
1605 }
1606
1607 static void __pool_dec(struct pool *pool)
1608 {
1609         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
1610         BUG_ON(!pool->ref_count);
1611         if (!--pool->ref_count)
1612                 __pool_destroy(pool);
1613 }
1614
1615 static struct pool *__pool_find(struct mapped_device *pool_md,
1616                                 struct block_device *metadata_dev,
1617                                 unsigned long block_size, char **error)
1618 {
1619         struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
1620
1621         if (pool) {
1622                 if (pool->pool_md != pool_md)
1623                         return ERR_PTR(-EBUSY);
1624                 __pool_inc(pool);
1625
1626         } else {
1627                 pool = __pool_table_lookup(pool_md);
1628                 if (pool) {
1629                         if (pool->md_dev != metadata_dev)
1630                                 return ERR_PTR(-EINVAL);
1631                         __pool_inc(pool);
1632
1633                 } else
1634                         pool = pool_create(pool_md, metadata_dev, block_size, error);
1635         }
1636
1637         return pool;
1638 }
1639
1640 /*----------------------------------------------------------------
1641  * Pool target methods
1642  *--------------------------------------------------------------*/
1643 static void pool_dtr(struct dm_target *ti)
1644 {
1645         struct pool_c *pt = ti->private;
1646
1647         mutex_lock(&dm_thin_pool_table.mutex);
1648
1649         unbind_control_target(pt->pool, ti);
1650         __pool_dec(pt->pool);
1651         dm_put_device(ti, pt->metadata_dev);
1652         dm_put_device(ti, pt->data_dev);
1653         kfree(pt);
1654
1655         mutex_unlock(&dm_thin_pool_table.mutex);
1656 }
1657
1658 struct pool_features {
1659         unsigned zero_new_blocks:1;
1660 };
1661
1662 static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
1663                                struct dm_target *ti)
1664 {
1665         int r;
1666         unsigned argc;
1667         const char *arg_name;
1668
1669         static struct dm_arg _args[] = {
1670                 {0, 1, "Invalid number of pool feature arguments"},
1671         };
1672
1673         /*
1674          * No feature arguments supplied.
1675          */
1676         if (!as->argc)
1677                 return 0;
1678
1679         r = dm_read_arg_group(_args, as, &argc, &ti->error);
1680         if (r)
1681                 return -EINVAL;
1682
1683         while (argc && !r) {
1684                 arg_name = dm_shift_arg(as);
1685                 argc--;
1686
1687                 if (!strcasecmp(arg_name, "skip_block_zeroing")) {
1688                         pf->zero_new_blocks = 0;
1689                         continue;
1690                 }
1691
1692                 ti->error = "Unrecognised pool feature requested";
1693                 r = -EINVAL;
1694         }
1695
1696         return r;
1697 }
1698
1699 /*
1700  * thin-pool <metadata dev> <data dev>
1701  *           <data block size (sectors)>
1702  *           <low water mark (blocks)>
1703  *           [<#feature args> [<arg>]*]
1704  *
1705  * Optional feature arguments are:
1706  *           skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
1707  */
1708 static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
1709 {
1710         int r;
1711         struct pool_c *pt;
1712         struct pool *pool;
1713         struct pool_features pf;
1714         struct dm_arg_set as;
1715         struct dm_dev *data_dev;
1716         unsigned long block_size;
1717         dm_block_t low_water_blocks;
1718         struct dm_dev *metadata_dev;
1719         sector_t metadata_dev_size;
1720
1721         /*
1722          * FIXME Remove validation from scope of lock.
1723          */
1724         mutex_lock(&dm_thin_pool_table.mutex);
1725
1726         if (argc < 4) {
1727                 ti->error = "Invalid argument count";
1728                 r = -EINVAL;
1729                 goto out_unlock;
1730         }
1731         as.argc = argc;
1732         as.argv = argv;
1733
1734         r = dm_get_device(ti, argv[0], FMODE_READ | FMODE_WRITE, &metadata_dev);
1735         if (r) {
1736                 ti->error = "Error opening metadata block device";
1737                 goto out_unlock;
1738         }
1739
1740         metadata_dev_size = i_size_read(metadata_dev->bdev->bd_inode) >> SECTOR_SHIFT;
1741         if (metadata_dev_size > METADATA_DEV_MAX_SECTORS) {
1742                 ti->error = "Metadata device is too large";
1743                 r = -EINVAL;
1744                 goto out_metadata;
1745         }
1746
1747         r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
1748         if (r) {
1749                 ti->error = "Error getting data device";
1750                 goto out_metadata;
1751         }
1752
1753         if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
1754             block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
1755             block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
1756             !is_power_of_2(block_size)) {
1757                 ti->error = "Invalid block size";
1758                 r = -EINVAL;
1759                 goto out;
1760         }
1761
1762         if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
1763                 ti->error = "Invalid low water mark";
1764                 r = -EINVAL;
1765                 goto out;
1766         }
1767
1768         /*
1769          * Set default pool features.
1770          */
1771         memset(&pf, 0, sizeof(pf));
1772         pf.zero_new_blocks = 1;
1773
1774         dm_consume_args(&as, 4);
1775         r = parse_pool_features(&as, &pf, ti);
1776         if (r)
1777                 goto out;
1778
1779         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
1780         if (!pt) {
1781                 r = -ENOMEM;
1782                 goto out;
1783         }
1784
1785         pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev,
1786                            block_size, &ti->error);
1787         if (IS_ERR(pool)) {
1788                 r = PTR_ERR(pool);
1789                 goto out_free_pt;
1790         }
1791
1792         pt->pool = pool;
1793         pt->ti = ti;
1794         pt->metadata_dev = metadata_dev;
1795         pt->data_dev = data_dev;
1796         pt->low_water_blocks = low_water_blocks;
1797         pt->zero_new_blocks = pf.zero_new_blocks;
1798         ti->num_flush_requests = 1;
1799         ti->num_discard_requests = 0;
1800         ti->private = pt;
1801
1802         pt->callbacks.congested_fn = pool_is_congested;
1803         dm_table_add_target_callbacks(ti->table, &pt->callbacks);
1804
1805         mutex_unlock(&dm_thin_pool_table.mutex);
1806
1807         return 0;
1808
1809 out_free_pt:
1810         kfree(pt);
1811 out:
1812         dm_put_device(ti, data_dev);
1813 out_metadata:
1814         dm_put_device(ti, metadata_dev);
1815 out_unlock:
1816         mutex_unlock(&dm_thin_pool_table.mutex);
1817
1818         return r;
1819 }
1820
1821 static int pool_map(struct dm_target *ti, struct bio *bio,
1822                     union map_info *map_context)
1823 {
1824         int r;
1825         struct pool_c *pt = ti->private;
1826         struct pool *pool = pt->pool;
1827         unsigned long flags;
1828
1829         /*
1830          * As this is a singleton target, ti->begin is always zero.
1831          */
1832         spin_lock_irqsave(&pool->lock, flags);
1833         bio->bi_bdev = pt->data_dev->bdev;
1834         r = DM_MAPIO_REMAPPED;
1835         spin_unlock_irqrestore(&pool->lock, flags);
1836
1837         return r;
1838 }
1839
1840 /*
1841  * Retrieves the number of blocks of the data device from
1842  * the superblock and compares it to the actual device size,
1843  * thus resizing the data device in case it has grown.
1844  *
1845  * This both copes with opening preallocated data devices in the ctr
1846  * being followed by a resume
1847  * -and-
1848  * calling the resume method individually after userspace has
1849  * grown the data device in reaction to a table event.
1850  */
1851 static int pool_preresume(struct dm_target *ti)
1852 {
1853         int r;
1854         struct pool_c *pt = ti->private;
1855         struct pool *pool = pt->pool;
1856         dm_block_t data_size, sb_data_size;
1857
1858         /*
1859          * Take control of the pool object.
1860          */
1861         r = bind_control_target(pool, ti);
1862         if (r)
1863                 return r;
1864
1865         data_size = ti->len >> pool->block_shift;
1866         r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
1867         if (r) {
1868                 DMERR("failed to retrieve data device size");
1869                 return r;
1870         }
1871
1872         if (data_size < sb_data_size) {
1873                 DMERR("pool target too small, is %llu blocks (expected %llu)",
1874                       data_size, sb_data_size);
1875                 return -EINVAL;
1876
1877         } else if (data_size > sb_data_size) {
1878                 r = dm_pool_resize_data_dev(pool->pmd, data_size);
1879                 if (r) {
1880                         DMERR("failed to resize data device");
1881                         return r;
1882                 }
1883
1884                 r = dm_pool_commit_metadata(pool->pmd);
1885                 if (r) {
1886                         DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1887                               __func__, r);
1888                         return r;
1889                 }
1890         }
1891
1892         return 0;
1893 }
1894
1895 static void pool_resume(struct dm_target *ti)
1896 {
1897         struct pool_c *pt = ti->private;
1898         struct pool *pool = pt->pool;
1899         unsigned long flags;
1900
1901         spin_lock_irqsave(&pool->lock, flags);
1902         pool->low_water_triggered = 0;
1903         pool->no_free_space = 0;
1904         __requeue_bios(pool);
1905         spin_unlock_irqrestore(&pool->lock, flags);
1906
1907         wake_worker(pool);
1908 }
1909
1910 static void pool_postsuspend(struct dm_target *ti)
1911 {
1912         int r;
1913         struct pool_c *pt = ti->private;
1914         struct pool *pool = pt->pool;
1915
1916         flush_workqueue(pool->wq);
1917
1918         r = dm_pool_commit_metadata(pool->pmd);
1919         if (r < 0) {
1920                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1921                       __func__, r);
1922                 /* FIXME: invalidate device? error the next FUA or FLUSH bio ?*/
1923         }
1924 }
1925
1926 static int check_arg_count(unsigned argc, unsigned args_required)
1927 {
1928         if (argc != args_required) {
1929                 DMWARN("Message received with %u arguments instead of %u.",
1930                        argc, args_required);
1931                 return -EINVAL;
1932         }
1933
1934         return 0;
1935 }
1936
1937 static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
1938 {
1939         if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
1940             *dev_id <= MAX_DEV_ID)
1941                 return 0;
1942
1943         if (warning)
1944                 DMWARN("Message received with invalid device id: %s", arg);
1945
1946         return -EINVAL;
1947 }
1948
1949 static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
1950 {
1951         dm_thin_id dev_id;
1952         int r;
1953
1954         r = check_arg_count(argc, 2);
1955         if (r)
1956                 return r;
1957
1958         r = read_dev_id(argv[1], &dev_id, 1);
1959         if (r)
1960                 return r;
1961
1962         r = dm_pool_create_thin(pool->pmd, dev_id);
1963         if (r) {
1964                 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
1965                        argv[1]);
1966                 return r;
1967         }
1968
1969         return 0;
1970 }
1971
1972 static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
1973 {
1974         dm_thin_id dev_id;
1975         dm_thin_id origin_dev_id;
1976         int r;
1977
1978         r = check_arg_count(argc, 3);
1979         if (r)
1980                 return r;
1981
1982         r = read_dev_id(argv[1], &dev_id, 1);
1983         if (r)
1984                 return r;
1985
1986         r = read_dev_id(argv[2], &origin_dev_id, 1);
1987         if (r)
1988                 return r;
1989
1990         r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
1991         if (r) {
1992                 DMWARN("Creation of new snapshot %s of device %s failed.",
1993                        argv[1], argv[2]);
1994                 return r;
1995         }
1996
1997         return 0;
1998 }
1999
2000 static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
2001 {
2002         dm_thin_id dev_id;
2003         int r;
2004
2005         r = check_arg_count(argc, 2);
2006         if (r)
2007                 return r;
2008
2009         r = read_dev_id(argv[1], &dev_id, 1);
2010         if (r)
2011                 return r;
2012
2013         r = dm_pool_delete_thin_device(pool->pmd, dev_id);
2014         if (r)
2015                 DMWARN("Deletion of thin device %s failed.", argv[1]);
2016
2017         return r;
2018 }
2019
2020 static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
2021 {
2022         dm_thin_id old_id, new_id;
2023         int r;
2024
2025         r = check_arg_count(argc, 3);
2026         if (r)
2027                 return r;
2028
2029         if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
2030                 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
2031                 return -EINVAL;
2032         }
2033
2034         if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
2035                 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
2036                 return -EINVAL;
2037         }
2038
2039         r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
2040         if (r) {
2041                 DMWARN("Failed to change transaction id from %s to %s.",
2042                        argv[1], argv[2]);
2043                 return r;
2044         }
2045
2046         return 0;
2047 }
2048
2049 /*
2050  * Messages supported:
2051  *   create_thin        <dev_id>
2052  *   create_snap        <dev_id> <origin_id>
2053  *   delete             <dev_id>
2054  *   trim               <dev_id> <new_size_in_sectors>
2055  *   set_transaction_id <current_trans_id> <new_trans_id>
2056  */
2057 static int pool_message(struct dm_target *ti, unsigned argc, char **argv)
2058 {
2059         int r = -EINVAL;
2060         struct pool_c *pt = ti->private;
2061         struct pool *pool = pt->pool;
2062
2063         if (!strcasecmp(argv[0], "create_thin"))
2064                 r = process_create_thin_mesg(argc, argv, pool);
2065
2066         else if (!strcasecmp(argv[0], "create_snap"))
2067                 r = process_create_snap_mesg(argc, argv, pool);
2068
2069         else if (!strcasecmp(argv[0], "delete"))
2070                 r = process_delete_mesg(argc, argv, pool);
2071
2072         else if (!strcasecmp(argv[0], "set_transaction_id"))
2073                 r = process_set_transaction_id_mesg(argc, argv, pool);
2074
2075         else
2076                 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
2077
2078         if (!r) {
2079                 r = dm_pool_commit_metadata(pool->pmd);
2080                 if (r)
2081                         DMERR("%s message: dm_pool_commit_metadata() failed, error = %d",
2082                               argv[0], r);
2083         }
2084
2085         return r;
2086 }
2087
2088 /*
2089  * Status line is:
2090  *    <transaction id> <used metadata sectors>/<total metadata sectors>
2091  *    <used data sectors>/<total data sectors> <held metadata root>
2092  */
2093 static void pool_status(struct dm_target *ti, status_type_t type,
2094                         char *result, unsigned maxlen)
2095 {
2096         int r;
2097         unsigned sz = 0;
2098         uint64_t transaction_id;
2099         dm_block_t nr_free_blocks_data;
2100         dm_block_t nr_free_blocks_metadata;
2101         dm_block_t nr_blocks_data;
2102         dm_block_t nr_blocks_metadata;
2103         dm_block_t held_root;
2104         char buf[BDEVNAME_SIZE];
2105         char buf2[BDEVNAME_SIZE];
2106         struct pool_c *pt = ti->private;
2107         struct pool *pool = pt->pool;
2108
2109         switch (type) {
2110         case STATUSTYPE_INFO:
2111                 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);
2112                 if (r) {
2113                         DMERR("dm_pool_get_metadata_transaction_id returned %d", r);
2114                         goto err;
2115                 }
2116
2117                 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);
2118                 if (r) {
2119                         DMERR("dm_pool_get_free_metadata_block_count returned %d", r);
2120                         goto err;
2121                 }
2122
2123                 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
2124                 if (r) {
2125                         DMERR("dm_pool_get_metadata_dev_size returned %d", r);
2126                         goto err;
2127                 }
2128
2129                 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);
2130                 if (r) {
2131                         DMERR("dm_pool_get_free_block_count returned %d", r);
2132                         goto err;
2133                 }
2134
2135                 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
2136                 if (r) {
2137                         DMERR("dm_pool_get_data_dev_size returned %d", r);
2138                         goto err;
2139                 }
2140
2141                 r = dm_pool_get_held_metadata_root(pool->pmd, &held_root);
2142                 if (r) {
2143                         DMERR("dm_pool_get_held_metadata_root returned %d", r);
2144                         goto err;
2145                 }
2146
2147                 DMEMIT("%llu %llu/%llu %llu/%llu ",
2148                        (unsigned long long)transaction_id,
2149                        (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
2150                        (unsigned long long)nr_blocks_metadata,
2151                        (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
2152                        (unsigned long long)nr_blocks_data);
2153
2154                 if (held_root)
2155                         DMEMIT("%llu", held_root);
2156                 else
2157                         DMEMIT("-");
2158
2159                 break;
2160
2161         case STATUSTYPE_TABLE:
2162                 DMEMIT("%s %s %lu %llu ",
2163                        format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
2164                        format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
2165                        (unsigned long)pool->sectors_per_block,
2166                        (unsigned long long)pt->low_water_blocks);
2167
2168                 DMEMIT("%u ", !pool->zero_new_blocks);
2169
2170                 if (!pool->zero_new_blocks)
2171                         DMEMIT("skip_block_zeroing ");
2172                 break;
2173         }
2174         return;
2175
2176 err:
2177         DMEMIT("Error");
2178 }
2179
2180 static int pool_iterate_devices(struct dm_target *ti,
2181                                 iterate_devices_callout_fn fn, void *data)
2182 {
2183         struct pool_c *pt = ti->private;
2184
2185         return fn(ti, pt->data_dev, 0, ti->len, data);
2186 }
2187
2188 static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2189                       struct bio_vec *biovec, int max_size)
2190 {
2191         struct pool_c *pt = ti->private;
2192         struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2193
2194         if (!q->merge_bvec_fn)
2195                 return max_size;
2196
2197         bvm->bi_bdev = pt->data_dev->bdev;
2198
2199         return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2200 }
2201
2202 static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
2203 {
2204         struct pool_c *pt = ti->private;
2205         struct pool *pool = pt->pool;
2206
2207         blk_limits_io_min(limits, 0);
2208         blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
2209 }
2210
2211 static struct target_type pool_target = {
2212         .name = "thin-pool",
2213         .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
2214                     DM_TARGET_IMMUTABLE,
2215         .version = {1, 0, 1},
2216         .module = THIS_MODULE,
2217         .ctr = pool_ctr,
2218         .dtr = pool_dtr,
2219         .map = pool_map,
2220         .postsuspend = pool_postsuspend,
2221         .preresume = pool_preresume,
2222         .resume = pool_resume,
2223         .message = pool_message,
2224         .status = pool_status,
2225         .merge = pool_merge,
2226         .iterate_devices = pool_iterate_devices,
2227         .io_hints = pool_io_hints,
2228 };
2229
2230 /*----------------------------------------------------------------
2231  * Thin target methods
2232  *--------------------------------------------------------------*/
2233 static void thin_dtr(struct dm_target *ti)
2234 {
2235         struct thin_c *tc = ti->private;
2236
2237         mutex_lock(&dm_thin_pool_table.mutex);
2238
2239         __pool_dec(tc->pool);
2240         dm_pool_close_thin_device(tc->td);
2241         dm_put_device(ti, tc->pool_dev);
2242         kfree(tc);
2243
2244         mutex_unlock(&dm_thin_pool_table.mutex);
2245 }
2246
2247 /*
2248  * Thin target parameters:
2249  *
2250  * <pool_dev> <dev_id>
2251  *
2252  * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
2253  * dev_id: the internal device identifier
2254  */
2255 static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
2256 {
2257         int r;
2258         struct thin_c *tc;
2259         struct dm_dev *pool_dev;
2260         struct mapped_device *pool_md;
2261
2262         mutex_lock(&dm_thin_pool_table.mutex);
2263
2264         if (argc != 2) {
2265                 ti->error = "Invalid argument count";
2266                 r = -EINVAL;
2267                 goto out_unlock;
2268         }
2269
2270         tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
2271         if (!tc) {
2272                 ti->error = "Out of memory";
2273                 r = -ENOMEM;
2274                 goto out_unlock;
2275         }
2276
2277         r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
2278         if (r) {
2279                 ti->error = "Error opening pool device";
2280                 goto bad_pool_dev;
2281         }
2282         tc->pool_dev = pool_dev;
2283
2284         if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
2285                 ti->error = "Invalid device id";
2286                 r = -EINVAL;
2287                 goto bad_common;
2288         }
2289
2290         pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
2291         if (!pool_md) {
2292                 ti->error = "Couldn't get pool mapped device";
2293                 r = -EINVAL;
2294                 goto bad_common;
2295         }
2296
2297         tc->pool = __pool_table_lookup(pool_md);
2298         if (!tc->pool) {
2299                 ti->error = "Couldn't find pool object";
2300                 r = -EINVAL;
2301                 goto bad_pool_lookup;
2302         }
2303         __pool_inc(tc->pool);
2304
2305         r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
2306         if (r) {
2307                 ti->error = "Couldn't open thin internal device";
2308                 goto bad_thin_open;
2309         }
2310
2311         ti->split_io = tc->pool->sectors_per_block;
2312         ti->num_flush_requests = 1;
2313         ti->num_discard_requests = 0;
2314         ti->discards_supported = 0;
2315
2316         dm_put(pool_md);
2317
2318         mutex_unlock(&dm_thin_pool_table.mutex);
2319
2320         return 0;
2321
2322 bad_thin_open:
2323         __pool_dec(tc->pool);
2324 bad_pool_lookup:
2325         dm_put(pool_md);
2326 bad_common:
2327         dm_put_device(ti, tc->pool_dev);
2328 bad_pool_dev:
2329         kfree(tc);
2330 out_unlock:
2331         mutex_unlock(&dm_thin_pool_table.mutex);
2332
2333         return r;
2334 }
2335
2336 static int thin_map(struct dm_target *ti, struct bio *bio,
2337                     union map_info *map_context)
2338 {
2339         bio->bi_sector -= ti->begin;
2340
2341         return thin_bio_map(ti, bio, map_context);
2342 }
2343
2344 static void thin_postsuspend(struct dm_target *ti)
2345 {
2346         if (dm_noflush_suspending(ti))
2347                 requeue_io((struct thin_c *)ti->private);
2348 }
2349
2350 /*
2351  * <nr mapped sectors> <highest mapped sector>
2352  */
2353 static void thin_status(struct dm_target *ti, status_type_t type,
2354                         char *result, unsigned maxlen)
2355 {
2356         int r;
2357         ssize_t sz = 0;
2358         dm_block_t mapped, highest;
2359         char buf[BDEVNAME_SIZE];
2360         struct thin_c *tc = ti->private;
2361
2362         if (!tc->td)
2363                 DMEMIT("-");
2364         else {
2365                 switch (type) {
2366                 case STATUSTYPE_INFO:
2367                         r = dm_thin_get_mapped_count(tc->td, &mapped);
2368                         if (r) {
2369                                 DMERR("dm_thin_get_mapped_count returned %d", r);
2370                                 goto err;
2371                         }
2372
2373                         r = dm_thin_get_highest_mapped_block(tc->td, &highest);
2374                         if (r < 0) {
2375                                 DMERR("dm_thin_get_highest_mapped_block returned %d", r);
2376                                 goto err;
2377                         }
2378
2379                         DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
2380                         if (r)
2381                                 DMEMIT("%llu", ((highest + 1) *
2382                                                 tc->pool->sectors_per_block) - 1);
2383                         else
2384                                 DMEMIT("-");
2385                         break;
2386
2387                 case STATUSTYPE_TABLE:
2388                         DMEMIT("%s %lu",
2389                                format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
2390                                (unsigned long) tc->dev_id);
2391                         break;
2392                 }
2393         }
2394
2395         return;
2396
2397 err:
2398         DMEMIT("Error");
2399 }
2400
2401 static int thin_iterate_devices(struct dm_target *ti,
2402                                 iterate_devices_callout_fn fn, void *data)
2403 {
2404         dm_block_t blocks;
2405         struct thin_c *tc = ti->private;
2406
2407         /*
2408          * We can't call dm_pool_get_data_dev_size() since that blocks.  So
2409          * we follow a more convoluted path through to the pool's target.
2410          */
2411         if (!tc->pool->ti)
2412                 return 0;       /* nothing is bound */
2413
2414         blocks = tc->pool->ti->len >> tc->pool->block_shift;
2415         if (blocks)
2416                 return fn(ti, tc->pool_dev, 0, tc->pool->sectors_per_block * blocks, data);
2417
2418         return 0;
2419 }
2420
2421 static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
2422 {
2423         struct thin_c *tc = ti->private;
2424
2425         blk_limits_io_min(limits, 0);
2426         blk_limits_io_opt(limits, tc->pool->sectors_per_block << SECTOR_SHIFT);
2427 }
2428
2429 static struct target_type thin_target = {
2430         .name = "thin",
2431         .version = {1, 0, 1},
2432         .module = THIS_MODULE,
2433         .ctr = thin_ctr,
2434         .dtr = thin_dtr,
2435         .map = thin_map,
2436         .postsuspend = thin_postsuspend,
2437         .status = thin_status,
2438         .iterate_devices = thin_iterate_devices,
2439         .io_hints = thin_io_hints,
2440 };
2441
2442 /*----------------------------------------------------------------*/
2443
2444 static int __init dm_thin_init(void)
2445 {
2446         int r;
2447
2448         pool_table_init();
2449
2450         r = dm_register_target(&thin_target);
2451         if (r)
2452                 return r;
2453
2454         r = dm_register_target(&pool_target);
2455         if (r)
2456                 dm_unregister_target(&thin_target);
2457
2458         return r;
2459 }
2460
2461 static void dm_thin_exit(void)
2462 {
2463         dm_unregister_target(&thin_target);
2464         dm_unregister_target(&pool_target);
2465 }
2466
2467 module_init(dm_thin_init);
2468 module_exit(dm_thin_exit);
2469
2470 MODULE_DESCRIPTION(DM_NAME "device-mapper thin provisioning target");
2471 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2472 MODULE_LICENSE("GPL");