dm snapshot: fix hung bios when copy error occurs
[pandora-kernel.git] / drivers / md / dm-snap.c
1 /*
2  * dm-snapshot.c
3  *
4  * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
5  *
6  * This file is released under the GPL.
7  */
8
9 #include <linux/blkdev.h>
10 #include <linux/device-mapper.h>
11 #include <linux/delay.h>
12 #include <linux/fs.h>
13 #include <linux/init.h>
14 #include <linux/kdev_t.h>
15 #include <linux/list.h>
16 #include <linux/mempool.h>
17 #include <linux/module.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/log2.h>
21 #include <linux/dm-kcopyd.h>
22
23 #include "dm-exception-store.h"
24
25 #define DM_MSG_PREFIX "snapshots"
26
27 static const char dm_snapshot_merge_target_name[] = "snapshot-merge";
28
29 #define dm_target_is_snapshot_merge(ti) \
30         ((ti)->type->name == dm_snapshot_merge_target_name)
31
32 /*
33  * The size of the mempool used to track chunks in use.
34  */
35 #define MIN_IOS 256
36
37 #define DM_TRACKED_CHUNK_HASH_SIZE      16
38 #define DM_TRACKED_CHUNK_HASH(x)        ((unsigned long)(x) & \
39                                          (DM_TRACKED_CHUNK_HASH_SIZE - 1))
40
41 struct dm_exception_table {
42         uint32_t hash_mask;
43         unsigned hash_shift;
44         struct list_head *table;
45 };
46
47 struct dm_snapshot {
48         struct rw_semaphore lock;
49
50         struct dm_dev *origin;
51         struct dm_dev *cow;
52
53         struct dm_target *ti;
54
55         /* List of snapshots per Origin */
56         struct list_head list;
57
58         /*
59          * You can't use a snapshot if this is 0 (e.g. if full).
60          * A snapshot-merge target never clears this.
61          */
62         int valid;
63
64         /* Origin writes don't trigger exceptions until this is set */
65         int active;
66
67         atomic_t pending_exceptions_count;
68
69         /* Protected by "lock" */
70         sector_t exception_start_sequence;
71
72         /* Protected by kcopyd single-threaded callback */
73         sector_t exception_complete_sequence;
74
75         /*
76          * A list of pending exceptions that completed out of order.
77          * Protected by kcopyd single-threaded callback.
78          */
79         struct list_head out_of_order_list;
80
81         mempool_t *pending_pool;
82
83         struct dm_exception_table pending;
84         struct dm_exception_table complete;
85
86         /*
87          * pe_lock protects all pending_exception operations and access
88          * as well as the snapshot_bios list.
89          */
90         spinlock_t pe_lock;
91
92         /* Chunks with outstanding reads */
93         spinlock_t tracked_chunk_lock;
94         mempool_t *tracked_chunk_pool;
95         struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
96
97         /* The on disk metadata handler */
98         struct dm_exception_store *store;
99
100         struct dm_kcopyd_client *kcopyd_client;
101
102         /* Wait for events based on state_bits */
103         unsigned long state_bits;
104
105         /* Range of chunks currently being merged. */
106         chunk_t first_merging_chunk;
107         int num_merging_chunks;
108
109         /*
110          * The merge operation failed if this flag is set.
111          * Failure modes are handled as follows:
112          * - I/O error reading the header
113          *      => don't load the target; abort.
114          * - Header does not have "valid" flag set
115          *      => use the origin; forget about the snapshot.
116          * - I/O error when reading exceptions
117          *      => don't load the target; abort.
118          *         (We can't use the intermediate origin state.)
119          * - I/O error while merging
120          *      => stop merging; set merge_failed; process I/O normally.
121          */
122         int merge_failed;
123
124         /*
125          * Incoming bios that overlap with chunks being merged must wait
126          * for them to be committed.
127          */
128         struct bio_list bios_queued_during_merge;
129 };
130
131 /*
132  * state_bits:
133  *   RUNNING_MERGE  - Merge operation is in progress.
134  *   SHUTDOWN_MERGE - Set to signal that merge needs to be stopped;
135  *                    cleared afterwards.
136  */
137 #define RUNNING_MERGE          0
138 #define SHUTDOWN_MERGE         1
139
140 struct dm_dev *dm_snap_origin(struct dm_snapshot *s)
141 {
142         return s->origin;
143 }
144 EXPORT_SYMBOL(dm_snap_origin);
145
146 struct dm_dev *dm_snap_cow(struct dm_snapshot *s)
147 {
148         return s->cow;
149 }
150 EXPORT_SYMBOL(dm_snap_cow);
151
152 static sector_t chunk_to_sector(struct dm_exception_store *store,
153                                 chunk_t chunk)
154 {
155         return chunk << store->chunk_shift;
156 }
157
158 static int bdev_equal(struct block_device *lhs, struct block_device *rhs)
159 {
160         /*
161          * There is only ever one instance of a particular block
162          * device so we can compare pointers safely.
163          */
164         return lhs == rhs;
165 }
166
167 struct dm_snap_pending_exception {
168         struct dm_exception e;
169
170         /*
171          * Origin buffers waiting for this to complete are held
172          * in a bio list
173          */
174         struct bio_list origin_bios;
175         struct bio_list snapshot_bios;
176
177         /* Pointer back to snapshot context */
178         struct dm_snapshot *snap;
179
180         /*
181          * 1 indicates the exception has already been sent to
182          * kcopyd.
183          */
184         int started;
185
186         /* There was copying error. */
187         int copy_error;
188
189         /* A sequence number, it is used for in-order completion. */
190         sector_t exception_sequence;
191
192         struct list_head out_of_order_entry;
193
194         /*
195          * For writing a complete chunk, bypassing the copy.
196          */
197         struct bio *full_bio;
198         bio_end_io_t *full_bio_end_io;
199         void *full_bio_private;
200 };
201
202 /*
203  * Hash table mapping origin volumes to lists of snapshots and
204  * a lock to protect it
205  */
206 static struct kmem_cache *exception_cache;
207 static struct kmem_cache *pending_cache;
208
209 struct dm_snap_tracked_chunk {
210         struct hlist_node node;
211         chunk_t chunk;
212 };
213
214 static struct kmem_cache *tracked_chunk_cache;
215
216 static struct dm_snap_tracked_chunk *track_chunk(struct dm_snapshot *s,
217                                                  chunk_t chunk)
218 {
219         struct dm_snap_tracked_chunk *c = mempool_alloc(s->tracked_chunk_pool,
220                                                         GFP_NOIO);
221         unsigned long flags;
222
223         c->chunk = chunk;
224
225         spin_lock_irqsave(&s->tracked_chunk_lock, flags);
226         hlist_add_head(&c->node,
227                        &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);
228         spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
229
230         return c;
231 }
232
233 static void stop_tracking_chunk(struct dm_snapshot *s,
234                                 struct dm_snap_tracked_chunk *c)
235 {
236         unsigned long flags;
237
238         spin_lock_irqsave(&s->tracked_chunk_lock, flags);
239         hlist_del(&c->node);
240         spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
241
242         mempool_free(c, s->tracked_chunk_pool);
243 }
244
245 static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)
246 {
247         struct dm_snap_tracked_chunk *c;
248         struct hlist_node *hn;
249         int found = 0;
250
251         spin_lock_irq(&s->tracked_chunk_lock);
252
253         hlist_for_each_entry(c, hn,
254             &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) {
255                 if (c->chunk == chunk) {
256                         found = 1;
257                         break;
258                 }
259         }
260
261         spin_unlock_irq(&s->tracked_chunk_lock);
262
263         return found;
264 }
265
266 /*
267  * This conflicting I/O is extremely improbable in the caller,
268  * so msleep(1) is sufficient and there is no need for a wait queue.
269  */
270 static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk)
271 {
272         while (__chunk_is_tracked(s, chunk))
273                 msleep(1);
274 }
275
276 /*
277  * One of these per registered origin, held in the snapshot_origins hash
278  */
279 struct origin {
280         /* The origin device */
281         struct block_device *bdev;
282
283         struct list_head hash_list;
284
285         /* List of snapshots for this origin */
286         struct list_head snapshots;
287 };
288
289 /*
290  * Size of the hash table for origin volumes. If we make this
291  * the size of the minors list then it should be nearly perfect
292  */
293 #define ORIGIN_HASH_SIZE 256
294 #define ORIGIN_MASK      0xFF
295 static struct list_head *_origins;
296 static struct rw_semaphore _origins_lock;
297
298 static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done);
299 static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock);
300 static uint64_t _pending_exceptions_done_count;
301
302 static int init_origin_hash(void)
303 {
304         int i;
305
306         _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head),
307                            GFP_KERNEL);
308         if (!_origins) {
309                 DMERR("unable to allocate memory");
310                 return -ENOMEM;
311         }
312
313         for (i = 0; i < ORIGIN_HASH_SIZE; i++)
314                 INIT_LIST_HEAD(_origins + i);
315         init_rwsem(&_origins_lock);
316
317         return 0;
318 }
319
320 static void exit_origin_hash(void)
321 {
322         kfree(_origins);
323 }
324
325 static unsigned origin_hash(struct block_device *bdev)
326 {
327         return bdev->bd_dev & ORIGIN_MASK;
328 }
329
330 static struct origin *__lookup_origin(struct block_device *origin)
331 {
332         struct list_head *ol;
333         struct origin *o;
334
335         ol = &_origins[origin_hash(origin)];
336         list_for_each_entry (o, ol, hash_list)
337                 if (bdev_equal(o->bdev, origin))
338                         return o;
339
340         return NULL;
341 }
342
343 static void __insert_origin(struct origin *o)
344 {
345         struct list_head *sl = &_origins[origin_hash(o->bdev)];
346         list_add_tail(&o->hash_list, sl);
347 }
348
349 /*
350  * _origins_lock must be held when calling this function.
351  * Returns number of snapshots registered using the supplied cow device, plus:
352  * snap_src - a snapshot suitable for use as a source of exception handover
353  * snap_dest - a snapshot capable of receiving exception handover.
354  * snap_merge - an existing snapshot-merge target linked to the same origin.
355  *   There can be at most one snapshot-merge target. The parameter is optional.
356  *
357  * Possible return values and states of snap_src and snap_dest.
358  *   0: NULL, NULL  - first new snapshot
359  *   1: snap_src, NULL - normal snapshot
360  *   2: snap_src, snap_dest  - waiting for handover
361  *   2: snap_src, NULL - handed over, waiting for old to be deleted
362  *   1: NULL, snap_dest - source got destroyed without handover
363  */
364 static int __find_snapshots_sharing_cow(struct dm_snapshot *snap,
365                                         struct dm_snapshot **snap_src,
366                                         struct dm_snapshot **snap_dest,
367                                         struct dm_snapshot **snap_merge)
368 {
369         struct dm_snapshot *s;
370         struct origin *o;
371         int count = 0;
372         int active;
373
374         o = __lookup_origin(snap->origin->bdev);
375         if (!o)
376                 goto out;
377
378         list_for_each_entry(s, &o->snapshots, list) {
379                 if (dm_target_is_snapshot_merge(s->ti) && snap_merge)
380                         *snap_merge = s;
381                 if (!bdev_equal(s->cow->bdev, snap->cow->bdev))
382                         continue;
383
384                 down_read(&s->lock);
385                 active = s->active;
386                 up_read(&s->lock);
387
388                 if (active) {
389                         if (snap_src)
390                                 *snap_src = s;
391                 } else if (snap_dest)
392                         *snap_dest = s;
393
394                 count++;
395         }
396
397 out:
398         return count;
399 }
400
401 /*
402  * On success, returns 1 if this snapshot is a handover destination,
403  * otherwise returns 0.
404  */
405 static int __validate_exception_handover(struct dm_snapshot *snap)
406 {
407         struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
408         struct dm_snapshot *snap_merge = NULL;
409
410         /* Does snapshot need exceptions handed over to it? */
411         if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest,
412                                           &snap_merge) == 2) ||
413             snap_dest) {
414                 snap->ti->error = "Snapshot cow pairing for exception "
415                                   "table handover failed";
416                 return -EINVAL;
417         }
418
419         /*
420          * If no snap_src was found, snap cannot become a handover
421          * destination.
422          */
423         if (!snap_src)
424                 return 0;
425
426         /*
427          * Non-snapshot-merge handover?
428          */
429         if (!dm_target_is_snapshot_merge(snap->ti))
430                 return 1;
431
432         /*
433          * Do not allow more than one merging snapshot.
434          */
435         if (snap_merge) {
436                 snap->ti->error = "A snapshot is already merging.";
437                 return -EINVAL;
438         }
439
440         if (!snap_src->store->type->prepare_merge ||
441             !snap_src->store->type->commit_merge) {
442                 snap->ti->error = "Snapshot exception store does not "
443                                   "support snapshot-merge.";
444                 return -EINVAL;
445         }
446
447         return 1;
448 }
449
450 static void __insert_snapshot(struct origin *o, struct dm_snapshot *s)
451 {
452         struct dm_snapshot *l;
453
454         /* Sort the list according to chunk size, largest-first smallest-last */
455         list_for_each_entry(l, &o->snapshots, list)
456                 if (l->store->chunk_size < s->store->chunk_size)
457                         break;
458         list_add_tail(&s->list, &l->list);
459 }
460
461 /*
462  * Make a note of the snapshot and its origin so we can look it
463  * up when the origin has a write on it.
464  *
465  * Also validate snapshot exception store handovers.
466  * On success, returns 1 if this registration is a handover destination,
467  * otherwise returns 0.
468  */
469 static int register_snapshot(struct dm_snapshot *snap)
470 {
471         struct origin *o, *new_o = NULL;
472         struct block_device *bdev = snap->origin->bdev;
473         int r = 0;
474
475         new_o = kmalloc(sizeof(*new_o), GFP_KERNEL);
476         if (!new_o)
477                 return -ENOMEM;
478
479         down_write(&_origins_lock);
480
481         r = __validate_exception_handover(snap);
482         if (r < 0) {
483                 kfree(new_o);
484                 goto out;
485         }
486
487         o = __lookup_origin(bdev);
488         if (o)
489                 kfree(new_o);
490         else {
491                 /* New origin */
492                 o = new_o;
493
494                 /* Initialise the struct */
495                 INIT_LIST_HEAD(&o->snapshots);
496                 o->bdev = bdev;
497
498                 __insert_origin(o);
499         }
500
501         __insert_snapshot(o, snap);
502
503 out:
504         up_write(&_origins_lock);
505
506         return r;
507 }
508
509 /*
510  * Move snapshot to correct place in list according to chunk size.
511  */
512 static void reregister_snapshot(struct dm_snapshot *s)
513 {
514         struct block_device *bdev = s->origin->bdev;
515
516         down_write(&_origins_lock);
517
518         list_del(&s->list);
519         __insert_snapshot(__lookup_origin(bdev), s);
520
521         up_write(&_origins_lock);
522 }
523
524 static void unregister_snapshot(struct dm_snapshot *s)
525 {
526         struct origin *o;
527
528         down_write(&_origins_lock);
529         o = __lookup_origin(s->origin->bdev);
530
531         list_del(&s->list);
532         if (o && list_empty(&o->snapshots)) {
533                 list_del(&o->hash_list);
534                 kfree(o);
535         }
536
537         up_write(&_origins_lock);
538 }
539
540 /*
541  * Implementation of the exception hash tables.
542  * The lowest hash_shift bits of the chunk number are ignored, allowing
543  * some consecutive chunks to be grouped together.
544  */
545 static int dm_exception_table_init(struct dm_exception_table *et,
546                                    uint32_t size, unsigned hash_shift)
547 {
548         unsigned int i;
549
550         et->hash_shift = hash_shift;
551         et->hash_mask = size - 1;
552         et->table = dm_vcalloc(size, sizeof(struct list_head));
553         if (!et->table)
554                 return -ENOMEM;
555
556         for (i = 0; i < size; i++)
557                 INIT_LIST_HEAD(et->table + i);
558
559         return 0;
560 }
561
562 static void dm_exception_table_exit(struct dm_exception_table *et,
563                                     struct kmem_cache *mem)
564 {
565         struct list_head *slot;
566         struct dm_exception *ex, *next;
567         int i, size;
568
569         size = et->hash_mask + 1;
570         for (i = 0; i < size; i++) {
571                 slot = et->table + i;
572
573                 list_for_each_entry_safe (ex, next, slot, hash_list)
574                         kmem_cache_free(mem, ex);
575         }
576
577         vfree(et->table);
578 }
579
580 static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk)
581 {
582         return (chunk >> et->hash_shift) & et->hash_mask;
583 }
584
585 static void dm_remove_exception(struct dm_exception *e)
586 {
587         list_del(&e->hash_list);
588 }
589
590 /*
591  * Return the exception data for a sector, or NULL if not
592  * remapped.
593  */
594 static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et,
595                                                 chunk_t chunk)
596 {
597         struct list_head *slot;
598         struct dm_exception *e;
599
600         slot = &et->table[exception_hash(et, chunk)];
601         list_for_each_entry (e, slot, hash_list)
602                 if (chunk >= e->old_chunk &&
603                     chunk <= e->old_chunk + dm_consecutive_chunk_count(e))
604                         return e;
605
606         return NULL;
607 }
608
609 static struct dm_exception *alloc_completed_exception(void)
610 {
611         struct dm_exception *e;
612
613         e = kmem_cache_alloc(exception_cache, GFP_NOIO);
614         if (!e)
615                 e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
616
617         return e;
618 }
619
620 static void free_completed_exception(struct dm_exception *e)
621 {
622         kmem_cache_free(exception_cache, e);
623 }
624
625 static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)
626 {
627         struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool,
628                                                              GFP_NOIO);
629
630         atomic_inc(&s->pending_exceptions_count);
631         pe->snap = s;
632
633         return pe;
634 }
635
636 static void free_pending_exception(struct dm_snap_pending_exception *pe)
637 {
638         struct dm_snapshot *s = pe->snap;
639
640         mempool_free(pe, s->pending_pool);
641         smp_mb__before_atomic_dec();
642         atomic_dec(&s->pending_exceptions_count);
643 }
644
645 static void dm_insert_exception(struct dm_exception_table *eh,
646                                 struct dm_exception *new_e)
647 {
648         struct list_head *l;
649         struct dm_exception *e = NULL;
650
651         l = &eh->table[exception_hash(eh, new_e->old_chunk)];
652
653         /* Add immediately if this table doesn't support consecutive chunks */
654         if (!eh->hash_shift)
655                 goto out;
656
657         /* List is ordered by old_chunk */
658         list_for_each_entry_reverse(e, l, hash_list) {
659                 /* Insert after an existing chunk? */
660                 if (new_e->old_chunk == (e->old_chunk +
661                                          dm_consecutive_chunk_count(e) + 1) &&
662                     new_e->new_chunk == (dm_chunk_number(e->new_chunk) +
663                                          dm_consecutive_chunk_count(e) + 1)) {
664                         dm_consecutive_chunk_count_inc(e);
665                         free_completed_exception(new_e);
666                         return;
667                 }
668
669                 /* Insert before an existing chunk? */
670                 if (new_e->old_chunk == (e->old_chunk - 1) &&
671                     new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) {
672                         dm_consecutive_chunk_count_inc(e);
673                         e->old_chunk--;
674                         e->new_chunk--;
675                         free_completed_exception(new_e);
676                         return;
677                 }
678
679                 if (new_e->old_chunk > e->old_chunk)
680                         break;
681         }
682
683 out:
684         list_add(&new_e->hash_list, e ? &e->hash_list : l);
685 }
686
687 /*
688  * Callback used by the exception stores to load exceptions when
689  * initialising.
690  */
691 static int dm_add_exception(void *context, chunk_t old, chunk_t new)
692 {
693         struct dm_snapshot *s = context;
694         struct dm_exception *e;
695
696         e = alloc_completed_exception();
697         if (!e)
698                 return -ENOMEM;
699
700         e->old_chunk = old;
701
702         /* Consecutive_count is implicitly initialised to zero */
703         e->new_chunk = new;
704
705         dm_insert_exception(&s->complete, e);
706
707         return 0;
708 }
709
710 /*
711  * Return a minimum chunk size of all snapshots that have the specified origin.
712  * Return zero if the origin has no snapshots.
713  */
714 static sector_t __minimum_chunk_size(struct origin *o)
715 {
716         struct dm_snapshot *snap;
717         unsigned chunk_size = 0;
718
719         if (o)
720                 list_for_each_entry(snap, &o->snapshots, list)
721                         chunk_size = min_not_zero(chunk_size,
722                                                   snap->store->chunk_size);
723
724         return chunk_size;
725 }
726
727 /*
728  * Hard coded magic.
729  */
730 static int calc_max_buckets(void)
731 {
732         /* use a fixed size of 2MB */
733         unsigned long mem = 2 * 1024 * 1024;
734         mem /= sizeof(struct list_head);
735
736         return mem;
737 }
738
739 /*
740  * Allocate room for a suitable hash table.
741  */
742 static int init_hash_tables(struct dm_snapshot *s)
743 {
744         sector_t hash_size, cow_dev_size, max_buckets;
745
746         /*
747          * Calculate based on the size of the original volume or
748          * the COW volume...
749          */
750         cow_dev_size = get_dev_size(s->cow->bdev);
751         max_buckets = calc_max_buckets();
752
753         hash_size = cow_dev_size >> s->store->chunk_shift;
754         hash_size = min(hash_size, max_buckets);
755
756         if (hash_size < 64)
757                 hash_size = 64;
758         hash_size = rounddown_pow_of_two(hash_size);
759         if (dm_exception_table_init(&s->complete, hash_size,
760                                     DM_CHUNK_CONSECUTIVE_BITS))
761                 return -ENOMEM;
762
763         /*
764          * Allocate hash table for in-flight exceptions
765          * Make this smaller than the real hash table
766          */
767         hash_size >>= 3;
768         if (hash_size < 64)
769                 hash_size = 64;
770
771         if (dm_exception_table_init(&s->pending, hash_size, 0)) {
772                 dm_exception_table_exit(&s->complete, exception_cache);
773                 return -ENOMEM;
774         }
775
776         return 0;
777 }
778
779 static void merge_shutdown(struct dm_snapshot *s)
780 {
781         clear_bit_unlock(RUNNING_MERGE, &s->state_bits);
782         smp_mb__after_clear_bit();
783         wake_up_bit(&s->state_bits, RUNNING_MERGE);
784 }
785
786 static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)
787 {
788         s->first_merging_chunk = 0;
789         s->num_merging_chunks = 0;
790
791         return bio_list_get(&s->bios_queued_during_merge);
792 }
793
794 /*
795  * Remove one chunk from the index of completed exceptions.
796  */
797 static int __remove_single_exception_chunk(struct dm_snapshot *s,
798                                            chunk_t old_chunk)
799 {
800         struct dm_exception *e;
801
802         e = dm_lookup_exception(&s->complete, old_chunk);
803         if (!e) {
804                 DMERR("Corruption detected: exception for block %llu is "
805                       "on disk but not in memory",
806                       (unsigned long long)old_chunk);
807                 return -EINVAL;
808         }
809
810         /*
811          * If this is the only chunk using this exception, remove exception.
812          */
813         if (!dm_consecutive_chunk_count(e)) {
814                 dm_remove_exception(e);
815                 free_completed_exception(e);
816                 return 0;
817         }
818
819         /*
820          * The chunk may be either at the beginning or the end of a
821          * group of consecutive chunks - never in the middle.  We are
822          * removing chunks in the opposite order to that in which they
823          * were added, so this should always be true.
824          * Decrement the consecutive chunk counter and adjust the
825          * starting point if necessary.
826          */
827         if (old_chunk == e->old_chunk) {
828                 e->old_chunk++;
829                 e->new_chunk++;
830         } else if (old_chunk != e->old_chunk +
831                    dm_consecutive_chunk_count(e)) {
832                 DMERR("Attempt to merge block %llu from the "
833                       "middle of a chunk range [%llu - %llu]",
834                       (unsigned long long)old_chunk,
835                       (unsigned long long)e->old_chunk,
836                       (unsigned long long)
837                       e->old_chunk + dm_consecutive_chunk_count(e));
838                 return -EINVAL;
839         }
840
841         dm_consecutive_chunk_count_dec(e);
842
843         return 0;
844 }
845
846 static void flush_bios(struct bio *bio);
847
848 static int remove_single_exception_chunk(struct dm_snapshot *s)
849 {
850         struct bio *b = NULL;
851         int r;
852         chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1;
853
854         down_write(&s->lock);
855
856         /*
857          * Process chunks (and associated exceptions) in reverse order
858          * so that dm_consecutive_chunk_count_dec() accounting works.
859          */
860         do {
861                 r = __remove_single_exception_chunk(s, old_chunk);
862                 if (r)
863                         goto out;
864         } while (old_chunk-- > s->first_merging_chunk);
865
866         b = __release_queued_bios_after_merge(s);
867
868 out:
869         up_write(&s->lock);
870         if (b)
871                 flush_bios(b);
872
873         return r;
874 }
875
876 static int origin_write_extent(struct dm_snapshot *merging_snap,
877                                sector_t sector, unsigned chunk_size);
878
879 static void merge_callback(int read_err, unsigned long write_err,
880                            void *context);
881
882 static uint64_t read_pending_exceptions_done_count(void)
883 {
884         uint64_t pending_exceptions_done;
885
886         spin_lock(&_pending_exceptions_done_spinlock);
887         pending_exceptions_done = _pending_exceptions_done_count;
888         spin_unlock(&_pending_exceptions_done_spinlock);
889
890         return pending_exceptions_done;
891 }
892
893 static void increment_pending_exceptions_done_count(void)
894 {
895         spin_lock(&_pending_exceptions_done_spinlock);
896         _pending_exceptions_done_count++;
897         spin_unlock(&_pending_exceptions_done_spinlock);
898
899         wake_up_all(&_pending_exceptions_done);
900 }
901
902 static void snapshot_merge_next_chunks(struct dm_snapshot *s)
903 {
904         int i, linear_chunks;
905         chunk_t old_chunk, new_chunk;
906         struct dm_io_region src, dest;
907         sector_t io_size;
908         uint64_t previous_count;
909
910         BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits));
911         if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits)))
912                 goto shut;
913
914         /*
915          * valid flag never changes during merge, so no lock required.
916          */
917         if (!s->valid) {
918                 DMERR("Snapshot is invalid: can't merge");
919                 goto shut;
920         }
921
922         linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk,
923                                                       &new_chunk);
924         if (linear_chunks <= 0) {
925                 if (linear_chunks < 0) {
926                         DMERR("Read error in exception store: "
927                               "shutting down merge");
928                         down_write(&s->lock);
929                         s->merge_failed = 1;
930                         up_write(&s->lock);
931                 }
932                 goto shut;
933         }
934
935         /* Adjust old_chunk and new_chunk to reflect start of linear region */
936         old_chunk = old_chunk + 1 - linear_chunks;
937         new_chunk = new_chunk + 1 - linear_chunks;
938
939         /*
940          * Use one (potentially large) I/O to copy all 'linear_chunks'
941          * from the exception store to the origin
942          */
943         io_size = linear_chunks * s->store->chunk_size;
944
945         dest.bdev = s->origin->bdev;
946         dest.sector = chunk_to_sector(s->store, old_chunk);
947         dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector);
948
949         src.bdev = s->cow->bdev;
950         src.sector = chunk_to_sector(s->store, new_chunk);
951         src.count = dest.count;
952
953         /*
954          * Reallocate any exceptions needed in other snapshots then
955          * wait for the pending exceptions to complete.
956          * Each time any pending exception (globally on the system)
957          * completes we are woken and repeat the process to find out
958          * if we can proceed.  While this may not seem a particularly
959          * efficient algorithm, it is not expected to have any
960          * significant impact on performance.
961          */
962         previous_count = read_pending_exceptions_done_count();
963         while (origin_write_extent(s, dest.sector, io_size)) {
964                 wait_event(_pending_exceptions_done,
965                            (read_pending_exceptions_done_count() !=
966                             previous_count));
967                 /* Retry after the wait, until all exceptions are done. */
968                 previous_count = read_pending_exceptions_done_count();
969         }
970
971         down_write(&s->lock);
972         s->first_merging_chunk = old_chunk;
973         s->num_merging_chunks = linear_chunks;
974         up_write(&s->lock);
975
976         /* Wait until writes to all 'linear_chunks' drain */
977         for (i = 0; i < linear_chunks; i++)
978                 __check_for_conflicting_io(s, old_chunk + i);
979
980         dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s);
981         return;
982
983 shut:
984         merge_shutdown(s);
985 }
986
987 static void error_bios(struct bio *bio);
988
989 static void merge_callback(int read_err, unsigned long write_err, void *context)
990 {
991         struct dm_snapshot *s = context;
992         struct bio *b = NULL;
993
994         if (read_err || write_err) {
995                 if (read_err)
996                         DMERR("Read error: shutting down merge.");
997                 else
998                         DMERR("Write error: shutting down merge.");
999                 goto shut;
1000         }
1001
1002         if (s->store->type->commit_merge(s->store,
1003                                          s->num_merging_chunks) < 0) {
1004                 DMERR("Write error in exception store: shutting down merge");
1005                 goto shut;
1006         }
1007
1008         if (remove_single_exception_chunk(s) < 0)
1009                 goto shut;
1010
1011         snapshot_merge_next_chunks(s);
1012
1013         return;
1014
1015 shut:
1016         down_write(&s->lock);
1017         s->merge_failed = 1;
1018         b = __release_queued_bios_after_merge(s);
1019         up_write(&s->lock);
1020         error_bios(b);
1021
1022         merge_shutdown(s);
1023 }
1024
1025 static void start_merge(struct dm_snapshot *s)
1026 {
1027         if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits))
1028                 snapshot_merge_next_chunks(s);
1029 }
1030
1031 static int wait_schedule(void *ptr)
1032 {
1033         schedule();
1034
1035         return 0;
1036 }
1037
1038 /*
1039  * Stop the merging process and wait until it finishes.
1040  */
1041 static void stop_merge(struct dm_snapshot *s)
1042 {
1043         set_bit(SHUTDOWN_MERGE, &s->state_bits);
1044         wait_on_bit(&s->state_bits, RUNNING_MERGE, wait_schedule,
1045                     TASK_UNINTERRUPTIBLE);
1046         clear_bit(SHUTDOWN_MERGE, &s->state_bits);
1047 }
1048
1049 /*
1050  * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
1051  */
1052 static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1053 {
1054         struct dm_snapshot *s;
1055         int i;
1056         int r = -EINVAL;
1057         char *origin_path, *cow_path;
1058         unsigned args_used, num_flush_requests = 1;
1059         fmode_t origin_mode = FMODE_READ;
1060
1061         if (argc != 4) {
1062                 ti->error = "requires exactly 4 arguments";
1063                 r = -EINVAL;
1064                 goto bad;
1065         }
1066
1067         if (dm_target_is_snapshot_merge(ti)) {
1068                 num_flush_requests = 2;
1069                 origin_mode = FMODE_WRITE;
1070         }
1071
1072         s = kmalloc(sizeof(*s), GFP_KERNEL);
1073         if (!s) {
1074                 ti->error = "Cannot allocate private snapshot structure";
1075                 r = -ENOMEM;
1076                 goto bad;
1077         }
1078
1079         origin_path = argv[0];
1080         argv++;
1081         argc--;
1082
1083         r = dm_get_device(ti, origin_path, origin_mode, &s->origin);
1084         if (r) {
1085                 ti->error = "Cannot get origin device";
1086                 goto bad_origin;
1087         }
1088
1089         cow_path = argv[0];
1090         argv++;
1091         argc--;
1092
1093         r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow);
1094         if (r) {
1095                 ti->error = "Cannot get COW device";
1096                 goto bad_cow;
1097         }
1098
1099         r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store);
1100         if (r) {
1101                 ti->error = "Couldn't create exception store";
1102                 r = -EINVAL;
1103                 goto bad_store;
1104         }
1105
1106         argv += args_used;
1107         argc -= args_used;
1108
1109         s->ti = ti;
1110         s->valid = 1;
1111         s->active = 0;
1112         atomic_set(&s->pending_exceptions_count, 0);
1113         s->exception_start_sequence = 0;
1114         s->exception_complete_sequence = 0;
1115         INIT_LIST_HEAD(&s->out_of_order_list);
1116         init_rwsem(&s->lock);
1117         INIT_LIST_HEAD(&s->list);
1118         spin_lock_init(&s->pe_lock);
1119         s->state_bits = 0;
1120         s->merge_failed = 0;
1121         s->first_merging_chunk = 0;
1122         s->num_merging_chunks = 0;
1123         bio_list_init(&s->bios_queued_during_merge);
1124
1125         /* Allocate hash table for COW data */
1126         if (init_hash_tables(s)) {
1127                 ti->error = "Unable to allocate hash table space";
1128                 r = -ENOMEM;
1129                 goto bad_hash_tables;
1130         }
1131
1132         s->kcopyd_client = dm_kcopyd_client_create();
1133         if (IS_ERR(s->kcopyd_client)) {
1134                 r = PTR_ERR(s->kcopyd_client);
1135                 ti->error = "Could not create kcopyd client";
1136                 goto bad_kcopyd;
1137         }
1138
1139         s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache);
1140         if (!s->pending_pool) {
1141                 ti->error = "Could not allocate mempool for pending exceptions";
1142                 r = -ENOMEM;
1143                 goto bad_pending_pool;
1144         }
1145
1146         s->tracked_chunk_pool = mempool_create_slab_pool(MIN_IOS,
1147                                                          tracked_chunk_cache);
1148         if (!s->tracked_chunk_pool) {
1149                 ti->error = "Could not allocate tracked_chunk mempool for "
1150                             "tracking reads";
1151                 goto bad_tracked_chunk_pool;
1152         }
1153
1154         for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
1155                 INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
1156
1157         spin_lock_init(&s->tracked_chunk_lock);
1158
1159         ti->private = s;
1160         ti->num_flush_requests = num_flush_requests;
1161
1162         /* Add snapshot to the list of snapshots for this origin */
1163         /* Exceptions aren't triggered till snapshot_resume() is called */
1164         r = register_snapshot(s);
1165         if (r == -ENOMEM) {
1166                 ti->error = "Snapshot origin struct allocation failed";
1167                 goto bad_load_and_register;
1168         } else if (r < 0) {
1169                 /* invalid handover, register_snapshot has set ti->error */
1170                 goto bad_load_and_register;
1171         }
1172
1173         /*
1174          * Metadata must only be loaded into one table at once, so skip this
1175          * if metadata will be handed over during resume.
1176          * Chunk size will be set during the handover - set it to zero to
1177          * ensure it's ignored.
1178          */
1179         if (r > 0) {
1180                 s->store->chunk_size = 0;
1181                 return 0;
1182         }
1183
1184         r = s->store->type->read_metadata(s->store, dm_add_exception,
1185                                           (void *)s);
1186         if (r < 0) {
1187                 ti->error = "Failed to read snapshot metadata";
1188                 goto bad_read_metadata;
1189         } else if (r > 0) {
1190                 s->valid = 0;
1191                 DMWARN("Snapshot is marked invalid.");
1192         }
1193
1194         if (!s->store->chunk_size) {
1195                 ti->error = "Chunk size not set";
1196                 goto bad_read_metadata;
1197         }
1198         ti->split_io = s->store->chunk_size;
1199
1200         return 0;
1201
1202 bad_read_metadata:
1203         unregister_snapshot(s);
1204
1205 bad_load_and_register:
1206         mempool_destroy(s->tracked_chunk_pool);
1207
1208 bad_tracked_chunk_pool:
1209         mempool_destroy(s->pending_pool);
1210
1211 bad_pending_pool:
1212         dm_kcopyd_client_destroy(s->kcopyd_client);
1213
1214 bad_kcopyd:
1215         dm_exception_table_exit(&s->pending, pending_cache);
1216         dm_exception_table_exit(&s->complete, exception_cache);
1217
1218 bad_hash_tables:
1219         dm_exception_store_destroy(s->store);
1220
1221 bad_store:
1222         dm_put_device(ti, s->cow);
1223
1224 bad_cow:
1225         dm_put_device(ti, s->origin);
1226
1227 bad_origin:
1228         kfree(s);
1229
1230 bad:
1231         return r;
1232 }
1233
1234 static void __free_exceptions(struct dm_snapshot *s)
1235 {
1236         dm_kcopyd_client_destroy(s->kcopyd_client);
1237         s->kcopyd_client = NULL;
1238
1239         dm_exception_table_exit(&s->pending, pending_cache);
1240         dm_exception_table_exit(&s->complete, exception_cache);
1241 }
1242
1243 static void __handover_exceptions(struct dm_snapshot *snap_src,
1244                                   struct dm_snapshot *snap_dest)
1245 {
1246         union {
1247                 struct dm_exception_table table_swap;
1248                 struct dm_exception_store *store_swap;
1249         } u;
1250
1251         /*
1252          * Swap all snapshot context information between the two instances.
1253          */
1254         u.table_swap = snap_dest->complete;
1255         snap_dest->complete = snap_src->complete;
1256         snap_src->complete = u.table_swap;
1257
1258         u.store_swap = snap_dest->store;
1259         snap_dest->store = snap_src->store;
1260         snap_src->store = u.store_swap;
1261
1262         snap_dest->store->snap = snap_dest;
1263         snap_src->store->snap = snap_src;
1264
1265         snap_dest->ti->split_io = snap_dest->store->chunk_size;
1266         snap_dest->valid = snap_src->valid;
1267
1268         /*
1269          * Set source invalid to ensure it receives no further I/O.
1270          */
1271         snap_src->valid = 0;
1272 }
1273
1274 static void snapshot_dtr(struct dm_target *ti)
1275 {
1276 #ifdef CONFIG_DM_DEBUG
1277         int i;
1278 #endif
1279         struct dm_snapshot *s = ti->private;
1280         struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
1281
1282         down_read(&_origins_lock);
1283         /* Check whether exception handover must be cancelled */
1284         (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
1285         if (snap_src && snap_dest && (s == snap_src)) {
1286                 down_write(&snap_dest->lock);
1287                 snap_dest->valid = 0;
1288                 up_write(&snap_dest->lock);
1289                 DMERR("Cancelling snapshot handover.");
1290         }
1291         up_read(&_origins_lock);
1292
1293         if (dm_target_is_snapshot_merge(ti))
1294                 stop_merge(s);
1295
1296         /* Prevent further origin writes from using this snapshot. */
1297         /* After this returns there can be no new kcopyd jobs. */
1298         unregister_snapshot(s);
1299
1300         while (atomic_read(&s->pending_exceptions_count))
1301                 msleep(1);
1302         /*
1303          * Ensure instructions in mempool_destroy aren't reordered
1304          * before atomic_read.
1305          */
1306         smp_mb();
1307
1308 #ifdef CONFIG_DM_DEBUG
1309         for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
1310                 BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));
1311 #endif
1312
1313         mempool_destroy(s->tracked_chunk_pool);
1314
1315         __free_exceptions(s);
1316
1317         mempool_destroy(s->pending_pool);
1318
1319         dm_exception_store_destroy(s->store);
1320
1321         dm_put_device(ti, s->cow);
1322
1323         dm_put_device(ti, s->origin);
1324
1325         kfree(s);
1326 }
1327
1328 /*
1329  * Flush a list of buffers.
1330  */
1331 static void flush_bios(struct bio *bio)
1332 {
1333         struct bio *n;
1334
1335         while (bio) {
1336                 n = bio->bi_next;
1337                 bio->bi_next = NULL;
1338                 generic_make_request(bio);
1339                 bio = n;
1340         }
1341 }
1342
1343 static int do_origin(struct dm_dev *origin, struct bio *bio);
1344
1345 /*
1346  * Flush a list of buffers.
1347  */
1348 static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio)
1349 {
1350         struct bio *n;
1351         int r;
1352
1353         while (bio) {
1354                 n = bio->bi_next;
1355                 bio->bi_next = NULL;
1356                 r = do_origin(s->origin, bio);
1357                 if (r == DM_MAPIO_REMAPPED)
1358                         generic_make_request(bio);
1359                 bio = n;
1360         }
1361 }
1362
1363 /*
1364  * Error a list of buffers.
1365  */
1366 static void error_bios(struct bio *bio)
1367 {
1368         struct bio *n;
1369
1370         while (bio) {
1371                 n = bio->bi_next;
1372                 bio->bi_next = NULL;
1373                 bio_io_error(bio);
1374                 bio = n;
1375         }
1376 }
1377
1378 static void __invalidate_snapshot(struct dm_snapshot *s, int err)
1379 {
1380         if (!s->valid)
1381                 return;
1382
1383         if (err == -EIO)
1384                 DMERR("Invalidating snapshot: Error reading/writing.");
1385         else if (err == -ENOMEM)
1386                 DMERR("Invalidating snapshot: Unable to allocate exception.");
1387
1388         if (s->store->type->drop_snapshot)
1389                 s->store->type->drop_snapshot(s->store);
1390
1391         s->valid = 0;
1392
1393         dm_table_event(s->ti->table);
1394 }
1395
1396 static void pending_complete(void *context, int success)
1397 {
1398         struct dm_snap_pending_exception *pe = context;
1399         struct dm_exception *e;
1400         struct dm_snapshot *s = pe->snap;
1401         struct bio *origin_bios = NULL;
1402         struct bio *snapshot_bios = NULL;
1403         struct bio *full_bio = NULL;
1404         int error = 0;
1405
1406         if (!success) {
1407                 /* Read/write error - snapshot is unusable */
1408                 down_write(&s->lock);
1409                 __invalidate_snapshot(s, -EIO);
1410                 error = 1;
1411                 goto out;
1412         }
1413
1414         e = alloc_completed_exception();
1415         if (!e) {
1416                 down_write(&s->lock);
1417                 __invalidate_snapshot(s, -ENOMEM);
1418                 error = 1;
1419                 goto out;
1420         }
1421         *e = pe->e;
1422
1423         down_write(&s->lock);
1424         if (!s->valid) {
1425                 free_completed_exception(e);
1426                 error = 1;
1427                 goto out;
1428         }
1429
1430         /* Check for conflicting reads */
1431         __check_for_conflicting_io(s, pe->e.old_chunk);
1432
1433         /*
1434          * Add a proper exception, and remove the
1435          * in-flight exception from the list.
1436          */
1437         dm_insert_exception(&s->complete, e);
1438
1439 out:
1440         dm_remove_exception(&pe->e);
1441         snapshot_bios = bio_list_get(&pe->snapshot_bios);
1442         origin_bios = bio_list_get(&pe->origin_bios);
1443         full_bio = pe->full_bio;
1444         if (full_bio) {
1445                 full_bio->bi_end_io = pe->full_bio_end_io;
1446                 full_bio->bi_private = pe->full_bio_private;
1447         }
1448         increment_pending_exceptions_done_count();
1449
1450         up_write(&s->lock);
1451
1452         /* Submit any pending write bios */
1453         if (error) {
1454                 if (full_bio)
1455                         bio_io_error(full_bio);
1456                 error_bios(snapshot_bios);
1457         } else {
1458                 if (full_bio)
1459                         bio_endio(full_bio, 0);
1460                 flush_bios(snapshot_bios);
1461         }
1462
1463         retry_origin_bios(s, origin_bios);
1464
1465         free_pending_exception(pe);
1466 }
1467
1468 static void complete_exception(struct dm_snap_pending_exception *pe)
1469 {
1470         struct dm_snapshot *s = pe->snap;
1471
1472         /* Update the metadata if we are persistent */
1473         s->store->type->commit_exception(s->store, &pe->e, !pe->copy_error,
1474                                          pending_complete, pe);
1475 }
1476
1477 /*
1478  * Called when the copy I/O has finished.  kcopyd actually runs
1479  * this code so don't block.
1480  */
1481 static void copy_callback(int read_err, unsigned long write_err, void *context)
1482 {
1483         struct dm_snap_pending_exception *pe = context;
1484         struct dm_snapshot *s = pe->snap;
1485
1486         pe->copy_error = read_err || write_err;
1487
1488         if (pe->exception_sequence == s->exception_complete_sequence) {
1489                 s->exception_complete_sequence++;
1490                 complete_exception(pe);
1491
1492                 while (!list_empty(&s->out_of_order_list)) {
1493                         pe = list_entry(s->out_of_order_list.next,
1494                                         struct dm_snap_pending_exception, out_of_order_entry);
1495                         if (pe->exception_sequence != s->exception_complete_sequence)
1496                                 break;
1497                         s->exception_complete_sequence++;
1498                         list_del(&pe->out_of_order_entry);
1499                         complete_exception(pe);
1500                 }
1501         } else {
1502                 struct list_head *lh;
1503                 struct dm_snap_pending_exception *pe2;
1504
1505                 list_for_each_prev(lh, &s->out_of_order_list) {
1506                         pe2 = list_entry(lh, struct dm_snap_pending_exception, out_of_order_entry);
1507                         if (pe2->exception_sequence < pe->exception_sequence)
1508                                 break;
1509                 }
1510                 list_add(&pe->out_of_order_entry, lh);
1511         }
1512 }
1513
1514 /*
1515  * Dispatches the copy operation to kcopyd.
1516  */
1517 static void start_copy(struct dm_snap_pending_exception *pe)
1518 {
1519         struct dm_snapshot *s = pe->snap;
1520         struct dm_io_region src, dest;
1521         struct block_device *bdev = s->origin->bdev;
1522         sector_t dev_size;
1523
1524         dev_size = get_dev_size(bdev);
1525
1526         src.bdev = bdev;
1527         src.sector = chunk_to_sector(s->store, pe->e.old_chunk);
1528         src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector);
1529
1530         dest.bdev = s->cow->bdev;
1531         dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);
1532         dest.count = src.count;
1533
1534         /* Hand over to kcopyd */
1535         dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);
1536 }
1537
1538 static void full_bio_end_io(struct bio *bio, int error)
1539 {
1540         void *callback_data = bio->bi_private;
1541
1542         dm_kcopyd_do_callback(callback_data, 0, error ? 1 : 0);
1543 }
1544
1545 static void start_full_bio(struct dm_snap_pending_exception *pe,
1546                            struct bio *bio)
1547 {
1548         struct dm_snapshot *s = pe->snap;
1549         void *callback_data;
1550
1551         pe->full_bio = bio;
1552         pe->full_bio_end_io = bio->bi_end_io;
1553         pe->full_bio_private = bio->bi_private;
1554
1555         callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client,
1556                                                    copy_callback, pe);
1557
1558         bio->bi_end_io = full_bio_end_io;
1559         bio->bi_private = callback_data;
1560
1561         generic_make_request(bio);
1562 }
1563
1564 static struct dm_snap_pending_exception *
1565 __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)
1566 {
1567         struct dm_exception *e = dm_lookup_exception(&s->pending, chunk);
1568
1569         if (!e)
1570                 return NULL;
1571
1572         return container_of(e, struct dm_snap_pending_exception, e);
1573 }
1574
1575 /*
1576  * Looks to see if this snapshot already has a pending exception
1577  * for this chunk, otherwise it allocates a new one and inserts
1578  * it into the pending table.
1579  *
1580  * NOTE: a write lock must be held on snap->lock before calling
1581  * this.
1582  */
1583 static struct dm_snap_pending_exception *
1584 __find_pending_exception(struct dm_snapshot *s,
1585                          struct dm_snap_pending_exception *pe, chunk_t chunk)
1586 {
1587         struct dm_snap_pending_exception *pe2;
1588
1589         pe2 = __lookup_pending_exception(s, chunk);
1590         if (pe2) {
1591                 free_pending_exception(pe);
1592                 return pe2;
1593         }
1594
1595         pe->e.old_chunk = chunk;
1596         bio_list_init(&pe->origin_bios);
1597         bio_list_init(&pe->snapshot_bios);
1598         pe->started = 0;
1599         pe->full_bio = NULL;
1600
1601         if (s->store->type->prepare_exception(s->store, &pe->e)) {
1602                 free_pending_exception(pe);
1603                 return NULL;
1604         }
1605
1606         pe->exception_sequence = s->exception_start_sequence++;
1607
1608         dm_insert_exception(&s->pending, &pe->e);
1609
1610         return pe;
1611 }
1612
1613 static void remap_exception(struct dm_snapshot *s, struct dm_exception *e,
1614                             struct bio *bio, chunk_t chunk)
1615 {
1616         bio->bi_bdev = s->cow->bdev;
1617         bio->bi_sector = chunk_to_sector(s->store,
1618                                          dm_chunk_number(e->new_chunk) +
1619                                          (chunk - e->old_chunk)) +
1620                                          (bio->bi_sector &
1621                                           s->store->chunk_mask);
1622 }
1623
1624 static int snapshot_map(struct dm_target *ti, struct bio *bio,
1625                         union map_info *map_context)
1626 {
1627         struct dm_exception *e;
1628         struct dm_snapshot *s = ti->private;
1629         int r = DM_MAPIO_REMAPPED;
1630         chunk_t chunk;
1631         struct dm_snap_pending_exception *pe = NULL;
1632
1633         if (bio->bi_rw & REQ_FLUSH) {
1634                 bio->bi_bdev = s->cow->bdev;
1635                 return DM_MAPIO_REMAPPED;
1636         }
1637
1638         chunk = sector_to_chunk(s->store, bio->bi_sector);
1639
1640         /* Full snapshots are not usable */
1641         /* To get here the table must be live so s->active is always set. */
1642         if (!s->valid)
1643                 return -EIO;
1644
1645         /* FIXME: should only take write lock if we need
1646          * to copy an exception */
1647         down_write(&s->lock);
1648
1649         if (!s->valid) {
1650                 r = -EIO;
1651                 goto out_unlock;
1652         }
1653
1654         /* If the block is already remapped - use that, else remap it */
1655         e = dm_lookup_exception(&s->complete, chunk);
1656         if (e) {
1657                 remap_exception(s, e, bio, chunk);
1658                 goto out_unlock;
1659         }
1660
1661         /*
1662          * Write to snapshot - higher level takes care of RW/RO
1663          * flags so we should only get this if we are
1664          * writeable.
1665          */
1666         if (bio_rw(bio) == WRITE) {
1667                 pe = __lookup_pending_exception(s, chunk);
1668                 if (!pe) {
1669                         up_write(&s->lock);
1670                         pe = alloc_pending_exception(s);
1671                         down_write(&s->lock);
1672
1673                         if (!s->valid) {
1674                                 free_pending_exception(pe);
1675                                 r = -EIO;
1676                                 goto out_unlock;
1677                         }
1678
1679                         e = dm_lookup_exception(&s->complete, chunk);
1680                         if (e) {
1681                                 free_pending_exception(pe);
1682                                 remap_exception(s, e, bio, chunk);
1683                                 goto out_unlock;
1684                         }
1685
1686                         pe = __find_pending_exception(s, pe, chunk);
1687                         if (!pe) {
1688                                 __invalidate_snapshot(s, -ENOMEM);
1689                                 r = -EIO;
1690                                 goto out_unlock;
1691                         }
1692                 }
1693
1694                 remap_exception(s, &pe->e, bio, chunk);
1695
1696                 r = DM_MAPIO_SUBMITTED;
1697
1698                 if (!pe->started &&
1699                     bio->bi_size == (s->store->chunk_size << SECTOR_SHIFT)) {
1700                         pe->started = 1;
1701                         up_write(&s->lock);
1702                         start_full_bio(pe, bio);
1703                         goto out;
1704                 }
1705
1706                 bio_list_add(&pe->snapshot_bios, bio);
1707
1708                 if (!pe->started) {
1709                         /* this is protected by snap->lock */
1710                         pe->started = 1;
1711                         up_write(&s->lock);
1712                         start_copy(pe);
1713                         goto out;
1714                 }
1715         } else {
1716                 bio->bi_bdev = s->origin->bdev;
1717                 map_context->ptr = track_chunk(s, chunk);
1718         }
1719
1720 out_unlock:
1721         up_write(&s->lock);
1722 out:
1723         return r;
1724 }
1725
1726 /*
1727  * A snapshot-merge target behaves like a combination of a snapshot
1728  * target and a snapshot-origin target.  It only generates new
1729  * exceptions in other snapshots and not in the one that is being
1730  * merged.
1731  *
1732  * For each chunk, if there is an existing exception, it is used to
1733  * redirect I/O to the cow device.  Otherwise I/O is sent to the origin,
1734  * which in turn might generate exceptions in other snapshots.
1735  * If merging is currently taking place on the chunk in question, the
1736  * I/O is deferred by adding it to s->bios_queued_during_merge.
1737  */
1738 static int snapshot_merge_map(struct dm_target *ti, struct bio *bio,
1739                               union map_info *map_context)
1740 {
1741         struct dm_exception *e;
1742         struct dm_snapshot *s = ti->private;
1743         int r = DM_MAPIO_REMAPPED;
1744         chunk_t chunk;
1745
1746         if (bio->bi_rw & REQ_FLUSH) {
1747                 if (!map_context->target_request_nr)
1748                         bio->bi_bdev = s->origin->bdev;
1749                 else
1750                         bio->bi_bdev = s->cow->bdev;
1751                 map_context->ptr = NULL;
1752                 return DM_MAPIO_REMAPPED;
1753         }
1754
1755         chunk = sector_to_chunk(s->store, bio->bi_sector);
1756
1757         down_write(&s->lock);
1758
1759         /* Full merging snapshots are redirected to the origin */
1760         if (!s->valid)
1761                 goto redirect_to_origin;
1762
1763         /* If the block is already remapped - use that */
1764         e = dm_lookup_exception(&s->complete, chunk);
1765         if (e) {
1766                 /* Queue writes overlapping with chunks being merged */
1767                 if (bio_rw(bio) == WRITE &&
1768                     chunk >= s->first_merging_chunk &&
1769                     chunk < (s->first_merging_chunk +
1770                              s->num_merging_chunks)) {
1771                         bio->bi_bdev = s->origin->bdev;
1772                         bio_list_add(&s->bios_queued_during_merge, bio);
1773                         r = DM_MAPIO_SUBMITTED;
1774                         goto out_unlock;
1775                 }
1776
1777                 remap_exception(s, e, bio, chunk);
1778
1779                 if (bio_rw(bio) == WRITE)
1780                         map_context->ptr = track_chunk(s, chunk);
1781                 goto out_unlock;
1782         }
1783
1784 redirect_to_origin:
1785         bio->bi_bdev = s->origin->bdev;
1786
1787         if (bio_rw(bio) == WRITE) {
1788                 up_write(&s->lock);
1789                 return do_origin(s->origin, bio);
1790         }
1791
1792 out_unlock:
1793         up_write(&s->lock);
1794
1795         return r;
1796 }
1797
1798 static int snapshot_end_io(struct dm_target *ti, struct bio *bio,
1799                            int error, union map_info *map_context)
1800 {
1801         struct dm_snapshot *s = ti->private;
1802         struct dm_snap_tracked_chunk *c = map_context->ptr;
1803
1804         if (c)
1805                 stop_tracking_chunk(s, c);
1806
1807         return 0;
1808 }
1809
1810 static void snapshot_merge_presuspend(struct dm_target *ti)
1811 {
1812         struct dm_snapshot *s = ti->private;
1813
1814         stop_merge(s);
1815 }
1816
1817 static int snapshot_preresume(struct dm_target *ti)
1818 {
1819         int r = 0;
1820         struct dm_snapshot *s = ti->private;
1821         struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
1822
1823         down_read(&_origins_lock);
1824         (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
1825         if (snap_src && snap_dest) {
1826                 down_read(&snap_src->lock);
1827                 if (s == snap_src) {
1828                         DMERR("Unable to resume snapshot source until "
1829                               "handover completes.");
1830                         r = -EINVAL;
1831                 } else if (!dm_suspended(snap_src->ti)) {
1832                         DMERR("Unable to perform snapshot handover until "
1833                               "source is suspended.");
1834                         r = -EINVAL;
1835                 }
1836                 up_read(&snap_src->lock);
1837         }
1838         up_read(&_origins_lock);
1839
1840         return r;
1841 }
1842
1843 static void snapshot_resume(struct dm_target *ti)
1844 {
1845         struct dm_snapshot *s = ti->private;
1846         struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
1847
1848         down_read(&_origins_lock);
1849         (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
1850         if (snap_src && snap_dest) {
1851                 down_write(&snap_src->lock);
1852                 down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING);
1853                 __handover_exceptions(snap_src, snap_dest);
1854                 up_write(&snap_dest->lock);
1855                 up_write(&snap_src->lock);
1856         }
1857         up_read(&_origins_lock);
1858
1859         /* Now we have correct chunk size, reregister */
1860         reregister_snapshot(s);
1861
1862         down_write(&s->lock);
1863         s->active = 1;
1864         up_write(&s->lock);
1865 }
1866
1867 static sector_t get_origin_minimum_chunksize(struct block_device *bdev)
1868 {
1869         sector_t min_chunksize;
1870
1871         down_read(&_origins_lock);
1872         min_chunksize = __minimum_chunk_size(__lookup_origin(bdev));
1873         up_read(&_origins_lock);
1874
1875         return min_chunksize;
1876 }
1877
1878 static void snapshot_merge_resume(struct dm_target *ti)
1879 {
1880         struct dm_snapshot *s = ti->private;
1881
1882         /*
1883          * Handover exceptions from existing snapshot.
1884          */
1885         snapshot_resume(ti);
1886
1887         /*
1888          * snapshot-merge acts as an origin, so set ti->split_io
1889          */
1890         ti->split_io = get_origin_minimum_chunksize(s->origin->bdev);
1891
1892         start_merge(s);
1893 }
1894
1895 static void snapshot_status(struct dm_target *ti, status_type_t type,
1896                             char *result, unsigned maxlen)
1897 {
1898         unsigned sz = 0;
1899         struct dm_snapshot *snap = ti->private;
1900
1901         switch (type) {
1902         case STATUSTYPE_INFO:
1903
1904                 down_write(&snap->lock);
1905
1906                 if (!snap->valid)
1907                         DMEMIT("Invalid");
1908                 else if (snap->merge_failed)
1909                         DMEMIT("Merge failed");
1910                 else {
1911                         if (snap->store->type->usage) {
1912                                 sector_t total_sectors, sectors_allocated,
1913                                          metadata_sectors;
1914                                 snap->store->type->usage(snap->store,
1915                                                          &total_sectors,
1916                                                          &sectors_allocated,
1917                                                          &metadata_sectors);
1918                                 DMEMIT("%llu/%llu %llu",
1919                                        (unsigned long long)sectors_allocated,
1920                                        (unsigned long long)total_sectors,
1921                                        (unsigned long long)metadata_sectors);
1922                         }
1923                         else
1924                                 DMEMIT("Unknown");
1925                 }
1926
1927                 up_write(&snap->lock);
1928
1929                 break;
1930
1931         case STATUSTYPE_TABLE:
1932                 /*
1933                  * kdevname returns a static pointer so we need
1934                  * to make private copies if the output is to
1935                  * make sense.
1936                  */
1937                 DMEMIT("%s %s", snap->origin->name, snap->cow->name);
1938                 snap->store->type->status(snap->store, type, result + sz,
1939                                           maxlen - sz);
1940                 break;
1941         }
1942 }
1943
1944 static int snapshot_iterate_devices(struct dm_target *ti,
1945                                     iterate_devices_callout_fn fn, void *data)
1946 {
1947         struct dm_snapshot *snap = ti->private;
1948         int r;
1949
1950         r = fn(ti, snap->origin, 0, ti->len, data);
1951
1952         if (!r)
1953                 r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data);
1954
1955         return r;
1956 }
1957
1958
1959 /*-----------------------------------------------------------------
1960  * Origin methods
1961  *---------------------------------------------------------------*/
1962
1963 /*
1964  * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any
1965  * supplied bio was ignored.  The caller may submit it immediately.
1966  * (No remapping actually occurs as the origin is always a direct linear
1967  * map.)
1968  *
1969  * If further exceptions are required, DM_MAPIO_SUBMITTED is returned
1970  * and any supplied bio is added to a list to be submitted once all
1971  * the necessary exceptions exist.
1972  */
1973 static int __origin_write(struct list_head *snapshots, sector_t sector,
1974                           struct bio *bio)
1975 {
1976         int r = DM_MAPIO_REMAPPED;
1977         struct dm_snapshot *snap;
1978         struct dm_exception *e;
1979         struct dm_snap_pending_exception *pe;
1980         struct dm_snap_pending_exception *pe_to_start_now = NULL;
1981         struct dm_snap_pending_exception *pe_to_start_last = NULL;
1982         chunk_t chunk;
1983
1984         /* Do all the snapshots on this origin */
1985         list_for_each_entry (snap, snapshots, list) {
1986                 /*
1987                  * Don't make new exceptions in a merging snapshot
1988                  * because it has effectively been deleted
1989                  */
1990                 if (dm_target_is_snapshot_merge(snap->ti))
1991                         continue;
1992
1993                 down_write(&snap->lock);
1994
1995                 /* Only deal with valid and active snapshots */
1996                 if (!snap->valid || !snap->active)
1997                         goto next_snapshot;
1998
1999                 /* Nothing to do if writing beyond end of snapshot */
2000                 if (sector >= dm_table_get_size(snap->ti->table))
2001                         goto next_snapshot;
2002
2003                 /*
2004                  * Remember, different snapshots can have
2005                  * different chunk sizes.
2006                  */
2007                 chunk = sector_to_chunk(snap->store, sector);
2008
2009                 /*
2010                  * Check exception table to see if block
2011                  * is already remapped in this snapshot
2012                  * and trigger an exception if not.
2013                  */
2014                 e = dm_lookup_exception(&snap->complete, chunk);
2015                 if (e)
2016                         goto next_snapshot;
2017
2018                 pe = __lookup_pending_exception(snap, chunk);
2019                 if (!pe) {
2020                         up_write(&snap->lock);
2021                         pe = alloc_pending_exception(snap);
2022                         down_write(&snap->lock);
2023
2024                         if (!snap->valid) {
2025                                 free_pending_exception(pe);
2026                                 goto next_snapshot;
2027                         }
2028
2029                         e = dm_lookup_exception(&snap->complete, chunk);
2030                         if (e) {
2031                                 free_pending_exception(pe);
2032                                 goto next_snapshot;
2033                         }
2034
2035                         pe = __find_pending_exception(snap, pe, chunk);
2036                         if (!pe) {
2037                                 __invalidate_snapshot(snap, -ENOMEM);
2038                                 goto next_snapshot;
2039                         }
2040                 }
2041
2042                 r = DM_MAPIO_SUBMITTED;
2043
2044                 /*
2045                  * If an origin bio was supplied, queue it to wait for the
2046                  * completion of this exception, and start this one last,
2047                  * at the end of the function.
2048                  */
2049                 if (bio) {
2050                         bio_list_add(&pe->origin_bios, bio);
2051                         bio = NULL;
2052
2053                         if (!pe->started) {
2054                                 pe->started = 1;
2055                                 pe_to_start_last = pe;
2056                         }
2057                 }
2058
2059                 if (!pe->started) {
2060                         pe->started = 1;
2061                         pe_to_start_now = pe;
2062                 }
2063
2064 next_snapshot:
2065                 up_write(&snap->lock);
2066
2067                 if (pe_to_start_now) {
2068                         start_copy(pe_to_start_now);
2069                         pe_to_start_now = NULL;
2070                 }
2071         }
2072
2073         /*
2074          * Submit the exception against which the bio is queued last,
2075          * to give the other exceptions a head start.
2076          */
2077         if (pe_to_start_last)
2078                 start_copy(pe_to_start_last);
2079
2080         return r;
2081 }
2082
2083 /*
2084  * Called on a write from the origin driver.
2085  */
2086 static int do_origin(struct dm_dev *origin, struct bio *bio)
2087 {
2088         struct origin *o;
2089         int r = DM_MAPIO_REMAPPED;
2090
2091         down_read(&_origins_lock);
2092         o = __lookup_origin(origin->bdev);
2093         if (o)
2094                 r = __origin_write(&o->snapshots, bio->bi_sector, bio);
2095         up_read(&_origins_lock);
2096
2097         return r;
2098 }
2099
2100 /*
2101  * Trigger exceptions in all non-merging snapshots.
2102  *
2103  * The chunk size of the merging snapshot may be larger than the chunk
2104  * size of some other snapshot so we may need to reallocate multiple
2105  * chunks in other snapshots.
2106  *
2107  * We scan all the overlapping exceptions in the other snapshots.
2108  * Returns 1 if anything was reallocated and must be waited for,
2109  * otherwise returns 0.
2110  *
2111  * size must be a multiple of merging_snap's chunk_size.
2112  */
2113 static int origin_write_extent(struct dm_snapshot *merging_snap,
2114                                sector_t sector, unsigned size)
2115 {
2116         int must_wait = 0;
2117         sector_t n;
2118         struct origin *o;
2119
2120         /*
2121          * The origin's __minimum_chunk_size() got stored in split_io
2122          * by snapshot_merge_resume().
2123          */
2124         down_read(&_origins_lock);
2125         o = __lookup_origin(merging_snap->origin->bdev);
2126         for (n = 0; n < size; n += merging_snap->ti->split_io)
2127                 if (__origin_write(&o->snapshots, sector + n, NULL) ==
2128                     DM_MAPIO_SUBMITTED)
2129                         must_wait = 1;
2130         up_read(&_origins_lock);
2131
2132         return must_wait;
2133 }
2134
2135 /*
2136  * Origin: maps a linear range of a device, with hooks for snapshotting.
2137  */
2138
2139 /*
2140  * Construct an origin mapping: <dev_path>
2141  * The context for an origin is merely a 'struct dm_dev *'
2142  * pointing to the real device.
2143  */
2144 static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2145 {
2146         int r;
2147         struct dm_dev *dev;
2148
2149         if (argc != 1) {
2150                 ti->error = "origin: incorrect number of arguments";
2151                 return -EINVAL;
2152         }
2153
2154         r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &dev);
2155         if (r) {
2156                 ti->error = "Cannot get target device";
2157                 return r;
2158         }
2159
2160         ti->private = dev;
2161         ti->num_flush_requests = 1;
2162
2163         return 0;
2164 }
2165
2166 static void origin_dtr(struct dm_target *ti)
2167 {
2168         struct dm_dev *dev = ti->private;
2169         dm_put_device(ti, dev);
2170 }
2171
2172 static int origin_map(struct dm_target *ti, struct bio *bio,
2173                       union map_info *map_context)
2174 {
2175         struct dm_dev *dev = ti->private;
2176         bio->bi_bdev = dev->bdev;
2177
2178         if (bio->bi_rw & REQ_FLUSH)
2179                 return DM_MAPIO_REMAPPED;
2180
2181         /* Only tell snapshots if this is a write */
2182         return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : DM_MAPIO_REMAPPED;
2183 }
2184
2185 /*
2186  * Set the target "split_io" field to the minimum of all the snapshots'
2187  * chunk sizes.
2188  */
2189 static void origin_resume(struct dm_target *ti)
2190 {
2191         struct dm_dev *dev = ti->private;
2192
2193         ti->split_io = get_origin_minimum_chunksize(dev->bdev);
2194 }
2195
2196 static void origin_status(struct dm_target *ti, status_type_t type,
2197                           char *result, unsigned maxlen)
2198 {
2199         struct dm_dev *dev = ti->private;
2200
2201         switch (type) {
2202         case STATUSTYPE_INFO:
2203                 result[0] = '\0';
2204                 break;
2205
2206         case STATUSTYPE_TABLE:
2207                 snprintf(result, maxlen, "%s", dev->name);
2208                 break;
2209         }
2210 }
2211
2212 static int origin_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2213                         struct bio_vec *biovec, int max_size)
2214 {
2215         struct dm_dev *dev = ti->private;
2216         struct request_queue *q = bdev_get_queue(dev->bdev);
2217
2218         if (!q->merge_bvec_fn)
2219                 return max_size;
2220
2221         bvm->bi_bdev = dev->bdev;
2222         bvm->bi_sector = bvm->bi_sector;
2223
2224         return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2225 }
2226
2227 static int origin_iterate_devices(struct dm_target *ti,
2228                                   iterate_devices_callout_fn fn, void *data)
2229 {
2230         struct dm_dev *dev = ti->private;
2231
2232         return fn(ti, dev, 0, ti->len, data);
2233 }
2234
2235 static struct target_type origin_target = {
2236         .name    = "snapshot-origin",
2237         .version = {1, 7, 2},
2238         .module  = THIS_MODULE,
2239         .ctr     = origin_ctr,
2240         .dtr     = origin_dtr,
2241         .map     = origin_map,
2242         .resume  = origin_resume,
2243         .status  = origin_status,
2244         .merge   = origin_merge,
2245         .iterate_devices = origin_iterate_devices,
2246 };
2247
2248 static struct target_type snapshot_target = {
2249         .name    = "snapshot",
2250         .version = {1, 10, 2},
2251         .module  = THIS_MODULE,
2252         .ctr     = snapshot_ctr,
2253         .dtr     = snapshot_dtr,
2254         .map     = snapshot_map,
2255         .end_io  = snapshot_end_io,
2256         .preresume  = snapshot_preresume,
2257         .resume  = snapshot_resume,
2258         .status  = snapshot_status,
2259         .iterate_devices = snapshot_iterate_devices,
2260 };
2261
2262 static struct target_type merge_target = {
2263         .name    = dm_snapshot_merge_target_name,
2264         .version = {1, 1, 0},
2265         .module  = THIS_MODULE,
2266         .ctr     = snapshot_ctr,
2267         .dtr     = snapshot_dtr,
2268         .map     = snapshot_merge_map,
2269         .end_io  = snapshot_end_io,
2270         .presuspend = snapshot_merge_presuspend,
2271         .preresume  = snapshot_preresume,
2272         .resume  = snapshot_merge_resume,
2273         .status  = snapshot_status,
2274         .iterate_devices = snapshot_iterate_devices,
2275 };
2276
2277 static int __init dm_snapshot_init(void)
2278 {
2279         int r;
2280
2281         r = dm_exception_store_init();
2282         if (r) {
2283                 DMERR("Failed to initialize exception stores");
2284                 return r;
2285         }
2286
2287         r = dm_register_target(&snapshot_target);
2288         if (r < 0) {
2289                 DMERR("snapshot target register failed %d", r);
2290                 goto bad_register_snapshot_target;
2291         }
2292
2293         r = dm_register_target(&origin_target);
2294         if (r < 0) {
2295                 DMERR("Origin target register failed %d", r);
2296                 goto bad_register_origin_target;
2297         }
2298
2299         r = dm_register_target(&merge_target);
2300         if (r < 0) {
2301                 DMERR("Merge target register failed %d", r);
2302                 goto bad_register_merge_target;
2303         }
2304
2305         r = init_origin_hash();
2306         if (r) {
2307                 DMERR("init_origin_hash failed.");
2308                 goto bad_origin_hash;
2309         }
2310
2311         exception_cache = KMEM_CACHE(dm_exception, 0);
2312         if (!exception_cache) {
2313                 DMERR("Couldn't create exception cache.");
2314                 r = -ENOMEM;
2315                 goto bad_exception_cache;
2316         }
2317
2318         pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0);
2319         if (!pending_cache) {
2320                 DMERR("Couldn't create pending cache.");
2321                 r = -ENOMEM;
2322                 goto bad_pending_cache;
2323         }
2324
2325         tracked_chunk_cache = KMEM_CACHE(dm_snap_tracked_chunk, 0);
2326         if (!tracked_chunk_cache) {
2327                 DMERR("Couldn't create cache to track chunks in use.");
2328                 r = -ENOMEM;
2329                 goto bad_tracked_chunk_cache;
2330         }
2331
2332         return 0;
2333
2334 bad_tracked_chunk_cache:
2335         kmem_cache_destroy(pending_cache);
2336 bad_pending_cache:
2337         kmem_cache_destroy(exception_cache);
2338 bad_exception_cache:
2339         exit_origin_hash();
2340 bad_origin_hash:
2341         dm_unregister_target(&merge_target);
2342 bad_register_merge_target:
2343         dm_unregister_target(&origin_target);
2344 bad_register_origin_target:
2345         dm_unregister_target(&snapshot_target);
2346 bad_register_snapshot_target:
2347         dm_exception_store_exit();
2348
2349         return r;
2350 }
2351
2352 static void __exit dm_snapshot_exit(void)
2353 {
2354         dm_unregister_target(&snapshot_target);
2355         dm_unregister_target(&origin_target);
2356         dm_unregister_target(&merge_target);
2357
2358         exit_origin_hash();
2359         kmem_cache_destroy(pending_cache);
2360         kmem_cache_destroy(exception_cache);
2361         kmem_cache_destroy(tracked_chunk_cache);
2362
2363         dm_exception_store_exit();
2364 }
2365
2366 /* Module hooks */
2367 module_init(dm_snapshot_init);
2368 module_exit(dm_snapshot_exit);
2369
2370 MODULE_DESCRIPTION(DM_NAME " snapshot target");
2371 MODULE_AUTHOR("Joe Thornber");
2372 MODULE_LICENSE("GPL");
2373 MODULE_ALIAS("dm-snapshot-origin");
2374 MODULE_ALIAS("dm-snapshot-merge");