[PATCH] Centralise definitions of sector_t and blkcnt_t
[pandora-kernel.git] / block / ll_rw_blk.c
index d388486..0f82e12 100644 (file)
@@ -39,6 +39,7 @@ static void blk_unplug_timeout(unsigned long data);
 static void drive_stat_acct(struct request *rq, int nr_sectors, int new_io);
 static void init_request_from_bio(struct request *req, struct bio *bio);
 static int __make_request(request_queue_t *q, struct bio *bio);
+static struct io_context *current_io_context(gfp_t gfp_flags, int node);
 
 /*
  * For the allocated request tables
@@ -55,11 +56,6 @@ static kmem_cache_t *requestq_cachep;
  */
 static kmem_cache_t *iocontext_cachep;
 
-static wait_queue_head_t congestion_wqh[2] = {
-               __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
-               __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
-       };
-
 /*
  * Controlling structure to kblockd
  */
@@ -111,35 +107,6 @@ static void blk_queue_congestion_threshold(struct request_queue *q)
        q->nr_congestion_off = nr;
 }
 
-/*
- * A queue has just exitted congestion.  Note this in the global counter of
- * congested queues, and wake up anyone who was waiting for requests to be
- * put back.
- */
-static void clear_queue_congested(request_queue_t *q, int rw)
-{
-       enum bdi_state bit;
-       wait_queue_head_t *wqh = &congestion_wqh[rw];
-
-       bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
-       clear_bit(bit, &q->backing_dev_info.state);
-       smp_mb__after_clear_bit();
-       if (waitqueue_active(wqh))
-               wake_up(wqh);
-}
-
-/*
- * A queue has just entered congestion.  Flag that in the queue's VM-visible
- * state flags and increment the global gounter of congested queues.
- */
-static void set_queue_congested(request_queue_t *q, int rw)
-{
-       enum bdi_state bit;
-
-       bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
-       set_bit(bit, &q->backing_dev_info.state);
-}
-
 /**
  * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
  * @bdev:      device
@@ -158,7 +125,6 @@ struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
                ret = &q->backing_dev_info;
        return ret;
 }
-
 EXPORT_SYMBOL(blk_get_backing_dev_info);
 
 void blk_queue_activity_fn(request_queue_t *q, activity_fn *fn, void *data)
@@ -166,7 +132,6 @@ void blk_queue_activity_fn(request_queue_t *q, activity_fn *fn, void *data)
        q->activity_fn = fn;
        q->activity_data = data;
 }
-
 EXPORT_SYMBOL(blk_queue_activity_fn);
 
 /**
@@ -277,13 +242,12 @@ void blk_queue_make_request(request_queue_t * q, make_request_fn * mfn)
 
 EXPORT_SYMBOL(blk_queue_make_request);
 
-static inline void rq_init(request_queue_t *q, struct request *rq)
+static void rq_init(request_queue_t *q, struct request *rq)
 {
        INIT_LIST_HEAD(&rq->queuelist);
        INIT_LIST_HEAD(&rq->donelist);
 
        rq->errors = 0;
-       rq->rq_status = RQ_ACTIVE;
        rq->bio = rq->biotail = NULL;
        INIT_HLIST_NODE(&rq->hash);
        RB_CLEAR_NODE(&rq->rb_node);
@@ -291,7 +255,6 @@ static inline void rq_init(request_queue_t *q, struct request *rq)
        rq->buffer = NULL;
        rq->ref_count = 1;
        rq->q = q;
-       rq->waiting = NULL;
        rq->special = NULL;
        rq->data_len = 0;
        rq->data = NULL;
@@ -451,8 +414,8 @@ static void queue_flush(request_queue_t *q, unsigned which)
        rq->cmd_flags = REQ_HARDBARRIER;
        rq_init(q, rq);
        rq->elevator_private = NULL;
+       rq->elevator_private2 = NULL;
        rq->rq_disk = q->bar_rq.rq_disk;
-       rq->rl = NULL;
        rq->end_io = end_io;
        q->prepare_flush_fn(q, rq);
 
@@ -479,7 +442,7 @@ static inline struct request *start_ordered(request_queue_t *q,
                rq->cmd_flags |= REQ_RW;
        rq->cmd_flags |= q->ordered & QUEUE_ORDERED_FUA ? REQ_FUA : 0;
        rq->elevator_private = NULL;
-       rq->rl = NULL;
+       rq->elevator_private2 = NULL;
        init_request_from_bio(rq, q->orig_bar_rq->bio);
        rq->end_io = bar_end_io;
 
@@ -591,8 +554,8 @@ static int flush_dry_bio_endio(struct bio *bio, unsigned int bytes, int error)
        return 0;
 }
 
-static inline int ordered_bio_endio(struct request *rq, struct bio *bio,
-                                   unsigned int nbytes, int error)
+static int ordered_bio_endio(struct request *rq, struct bio *bio,
+                            unsigned int nbytes, int error)
 {
        request_queue_t *q = rq->q;
        bio_end_io_t *endio;
@@ -841,12 +804,7 @@ EXPORT_SYMBOL(blk_queue_dma_alignment);
  **/
 struct request *blk_queue_find_tag(request_queue_t *q, int tag)
 {
-       struct blk_queue_tag *bqt = q->queue_tags;
-
-       if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
-               return NULL;
-
-       return bqt->tag_index[tag];
+       return blk_map_queue_find_tag(q->queue_tags, tag);
 }
 
 EXPORT_SYMBOL(blk_queue_find_tag);
@@ -1172,11 +1130,16 @@ int blk_queue_start_tag(request_queue_t *q, struct request *rq)
                BUG();
        }
 
-       tag = find_first_zero_bit(bqt->tag_map, bqt->max_depth);
-       if (tag >= bqt->max_depth)
-               return 1;
+       /*
+        * Protect against shared tag maps, as we may not have exclusive
+        * access to the tag map.
+        */
+       do {
+               tag = find_first_zero_bit(bqt->tag_map, bqt->max_depth);
+               if (tag >= bqt->max_depth)
+                       return 1;
 
-       __set_bit(tag, bqt->tag_map);
+       } while (test_and_set_bit(tag, bqt->tag_map));
 
        rq->cmd_flags |= REQ_QUEUED;
        rq->tag = tag;
@@ -2004,9 +1967,8 @@ static inline void blk_free_request(request_queue_t *q, struct request *rq)
        mempool_free(rq, q->rq.rq_pool);
 }
 
-static inline struct request *
-blk_alloc_request(request_queue_t *q, int rw, struct bio *bio,
-                 int priv, gfp_t gfp_mask)
+static struct request *
+blk_alloc_request(request_queue_t *q, int rw, int priv, gfp_t gfp_mask)
 {
        struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
 
@@ -2017,10 +1979,10 @@ blk_alloc_request(request_queue_t *q, int rw, struct bio *bio,
         * first three bits are identical in rq->cmd_flags and bio->bi_rw,
         * see bio.h and blkdev.h
         */
-       rq->cmd_flags = rw;
+       rq->cmd_flags = rw | REQ_ALLOCED;
 
        if (priv) {
-               if (unlikely(elv_set_request(q, rq, bio, gfp_mask))) {
+               if (unlikely(elv_set_request(q, rq, gfp_mask))) {
                        mempool_free(rq, q->rq.rq_pool);
                        return NULL;
                }
@@ -2069,7 +2031,7 @@ static void __freed_request(request_queue_t *q, int rw)
        struct request_list *rl = &q->rq;
 
        if (rl->count[rw] < queue_congestion_off_threshold(q))
-               clear_queue_congested(q, rw);
+               blk_clear_queue_congested(q, rw);
 
        if (rl->count[rw] + 1 <= q->nr_requests) {
                if (waitqueue_active(&rl->wait[rw]))
@@ -2111,13 +2073,13 @@ static struct request *get_request(request_queue_t *q, int rw, struct bio *bio,
        struct io_context *ioc = NULL;
        int may_queue, priv;
 
-       may_queue = elv_may_queue(q, rw, bio);
+       may_queue = elv_may_queue(q, rw);
        if (may_queue == ELV_MQUEUE_NO)
                goto rq_starved;
 
        if (rl->count[rw]+1 >= queue_congestion_on_threshold(q)) {
                if (rl->count[rw]+1 >= q->nr_requests) {
-                       ioc = current_io_context(GFP_ATOMIC);
+                       ioc = current_io_context(GFP_ATOMIC, q->node);
                        /*
                         * The queue will fill after this allocation, so set
                         * it as full, and mark this process as "batching".
@@ -2139,7 +2101,7 @@ static struct request *get_request(request_queue_t *q, int rw, struct bio *bio,
                                }
                        }
                }
-               set_queue_congested(q, rw);
+               blk_set_queue_congested(q, rw);
        }
 
        /*
@@ -2159,7 +2121,7 @@ static struct request *get_request(request_queue_t *q, int rw, struct bio *bio,
 
        spin_unlock_irq(q->queue_lock);
 
-       rq = blk_alloc_request(q, rw, bio, priv, gfp_mask);
+       rq = blk_alloc_request(q, rw, priv, gfp_mask);
        if (unlikely(!rq)) {
                /*
                 * Allocation failed presumably due to memory. Undo anything
@@ -2195,7 +2157,6 @@ rq_starved:
                ioc->nr_batch_requests--;
        
        rq_init(q, rq);
-       rq->rl = rl;
 
        blk_add_trace_generic(q, bio, rw, BLK_TA_GETRQ);
 out:
@@ -2238,7 +2199,7 @@ static struct request *get_request_wait(request_queue_t *q, int rw,
                         * up to a big batch of them for a small period time.
                         * See ioc_batching, ioc_set_batching
                         */
-                       ioc = current_io_context(GFP_NOIO);
+                       ioc = current_io_context(GFP_NOIO, q->node);
                        ioc_set_batching(q, ioc);
 
                        spin_lock_irq(q->queue_lock);
@@ -2269,6 +2230,25 @@ struct request *blk_get_request(request_queue_t *q, int rw, gfp_t gfp_mask)
 }
 EXPORT_SYMBOL(blk_get_request);
 
+/**
+ * blk_start_queueing - initiate dispatch of requests to device
+ * @q:         request queue to kick into gear
+ *
+ * This is basically a helper to remove the need to know whether a queue
+ * is plugged or not if someone just wants to initiate dispatch of requests
+ * for this queue.
+ *
+ * The queue lock must be held with interrupts disabled.
+ */
+void blk_start_queueing(request_queue_t *q)
+{
+       if (!blk_queue_plugged(q))
+               q->request_fn(q);
+       else
+               __generic_unplug_device(q);
+}
+EXPORT_SYMBOL(blk_start_queueing);
+
 /**
  * blk_requeue_request - put a request back on queue
  * @q:         request queue where request should be inserted
@@ -2336,16 +2316,90 @@ void blk_insert_request(request_queue_t *q, struct request *rq,
 
        drive_stat_acct(rq, rq->nr_sectors, 1);
        __elv_add_request(q, rq, where, 0);
-
-       if (blk_queue_plugged(q))
-               __generic_unplug_device(q);
-       else
-               q->request_fn(q);
+       blk_start_queueing(q);
        spin_unlock_irqrestore(q->queue_lock, flags);
 }
 
 EXPORT_SYMBOL(blk_insert_request);
 
+static int __blk_rq_unmap_user(struct bio *bio)
+{
+       int ret = 0;
+
+       if (bio) {
+               if (bio_flagged(bio, BIO_USER_MAPPED))
+                       bio_unmap_user(bio);
+               else
+                       ret = bio_uncopy_user(bio);
+       }
+
+       return ret;
+}
+
+static int __blk_rq_map_user(request_queue_t *q, struct request *rq,
+                            void __user *ubuf, unsigned int len)
+{
+       unsigned long uaddr;
+       struct bio *bio, *orig_bio;
+       int reading, ret;
+
+       reading = rq_data_dir(rq) == READ;
+
+       /*
+        * if alignment requirement is satisfied, map in user pages for
+        * direct dma. else, set up kernel bounce buffers
+        */
+       uaddr = (unsigned long) ubuf;
+       if (!(uaddr & queue_dma_alignment(q)) && !(len & queue_dma_alignment(q)))
+               bio = bio_map_user(q, NULL, uaddr, len, reading);
+       else
+               bio = bio_copy_user(q, uaddr, len, reading);
+
+       if (IS_ERR(bio)) {
+               return PTR_ERR(bio);
+       }
+
+       orig_bio = bio;
+       blk_queue_bounce(q, &bio);
+       /*
+        * We link the bounce buffer in and could have to traverse it
+        * later so we have to get a ref to prevent it from being freed
+        */
+       bio_get(bio);
+
+       /*
+        * for most (all? don't know of any) queues we could
+        * skip grabbing the queue lock here. only drivers with
+        * funky private ->back_merge_fn() function could be
+        * problematic.
+        */
+       spin_lock_irq(q->queue_lock);
+       if (!rq->bio)
+               blk_rq_bio_prep(q, rq, bio);
+       else if (!q->back_merge_fn(q, rq, bio)) {
+               ret = -EINVAL;
+               spin_unlock_irq(q->queue_lock);
+               goto unmap_bio;
+       } else {
+               rq->biotail->bi_next = bio;
+               rq->biotail = bio;
+
+               rq->nr_sectors += bio_sectors(bio);
+               rq->hard_nr_sectors = rq->nr_sectors;
+               rq->data_len += bio->bi_size;
+       }
+       spin_unlock_irq(q->queue_lock);
+
+       return bio->bi_size;
+
+unmap_bio:
+       /* if it was boucned we must call the end io function */
+       bio_endio(bio, bio->bi_size, 0);
+       __blk_rq_unmap_user(orig_bio);
+       bio_put(bio);
+       return ret;
+}
+
 /**
  * blk_rq_map_user - map user data to a request, for REQ_BLOCK_PC usage
  * @q:         request queue where request should be inserted
@@ -2367,42 +2421,44 @@ EXPORT_SYMBOL(blk_insert_request);
  *    unmapping.
  */
 int blk_rq_map_user(request_queue_t *q, struct request *rq, void __user *ubuf,
-                   unsigned int len)
+                   unsigned long len)
 {
-       unsigned long uaddr;
-       struct bio *bio;
-       int reading;
+       unsigned long bytes_read = 0;
+       int ret;
 
        if (len > (q->max_hw_sectors << 9))
                return -EINVAL;
        if (!len || !ubuf)
                return -EINVAL;
 
-       reading = rq_data_dir(rq) == READ;
+       while (bytes_read != len) {
+               unsigned long map_len, end, start;
 
-       /*
-        * if alignment requirement is satisfied, map in user pages for
-        * direct dma. else, set up kernel bounce buffers
-        */
-       uaddr = (unsigned long) ubuf;
-       if (!(uaddr & queue_dma_alignment(q)) && !(len & queue_dma_alignment(q)))
-               bio = bio_map_user(q, NULL, uaddr, len, reading);
-       else
-               bio = bio_copy_user(q, uaddr, len, reading);
+               map_len = min_t(unsigned long, len - bytes_read, BIO_MAX_SIZE);
+               end = ((unsigned long)ubuf + map_len + PAGE_SIZE - 1)
+                                                               >> PAGE_SHIFT;
+               start = (unsigned long)ubuf >> PAGE_SHIFT;
 
-       if (!IS_ERR(bio)) {
-               rq->bio = rq->biotail = bio;
-               blk_rq_bio_prep(q, rq, bio);
+               /*
+                * A bad offset could cause us to require BIO_MAX_PAGES + 1
+                * pages. If this happens we just lower the requested
+                * mapping len by a page so that we can fit
+                */
+               if (end - start > BIO_MAX_PAGES)
+                       map_len -= PAGE_SIZE;
 
-               rq->buffer = rq->data = NULL;
-               rq->data_len = len;
-               return 0;
+               ret = __blk_rq_map_user(q, rq, ubuf, map_len);
+               if (ret < 0)
+                       goto unmap_rq;
+               bytes_read += ret;
+               ubuf += ret;
        }
 
-       /*
-        * bio is the err-ptr
-        */
-       return PTR_ERR(bio);
+       rq->buffer = rq->data = NULL;
+       return 0;
+unmap_rq:
+       blk_rq_unmap_user(rq);
+       return ret;
 }
 
 EXPORT_SYMBOL(blk_rq_map_user);
@@ -2428,7 +2484,7 @@ EXPORT_SYMBOL(blk_rq_map_user);
  *    unmapping.
  */
 int blk_rq_map_user_iov(request_queue_t *q, struct request *rq,
-                       struct sg_iovec *iov, int iov_count)
+                       struct sg_iovec *iov, int iov_count, unsigned int len)
 {
        struct bio *bio;
 
@@ -2442,10 +2498,15 @@ int blk_rq_map_user_iov(request_queue_t *q, struct request *rq,
        if (IS_ERR(bio))
                return PTR_ERR(bio);
 
-       rq->bio = rq->biotail = bio;
+       if (bio->bi_size != len) {
+               bio_endio(bio, bio->bi_size, 0);
+               bio_unmap_user(bio);
+               return -EINVAL;
+       }
+
+       bio_get(bio);
        blk_rq_bio_prep(q, rq, bio);
        rq->buffer = rq->data = NULL;
-       rq->data_len = bio->bi_size;
        return 0;
 }
 
@@ -2453,23 +2514,26 @@ EXPORT_SYMBOL(blk_rq_map_user_iov);
 
 /**
  * blk_rq_unmap_user - unmap a request with user data
- * @bio:       bio to be unmapped
- * @ulen:      length of user buffer
+ * @rq:                rq to be unmapped
  *
  * Description:
- *    Unmap a bio previously mapped by blk_rq_map_user().
+ *    Unmap a rq previously mapped by blk_rq_map_user().
+ *    rq->bio must be set to the original head of the request.
  */
-int blk_rq_unmap_user(struct bio *bio, unsigned int ulen)
+int blk_rq_unmap_user(struct request *rq)
 {
-       int ret = 0;
+       struct bio *bio, *mapped_bio;
 
-       if (bio) {
-               if (bio_flagged(bio, BIO_USER_MAPPED))
-                       bio_unmap_user(bio);
+       while ((bio = rq->bio)) {
+               if (bio_flagged(bio, BIO_BOUNCED))
+                       mapped_bio = bio->bi_private;
                else
-                       ret = bio_uncopy_user(bio);
-       }
+                       mapped_bio = bio;
 
+               __blk_rq_unmap_user(mapped_bio);
+               rq->bio = bio->bi_next;
+               bio_put(bio);
+       }
        return 0;
 }
 
@@ -2500,11 +2564,8 @@ int blk_rq_map_kern(request_queue_t *q, struct request *rq, void *kbuf,
        if (rq_data_dir(rq) == WRITE)
                bio->bi_rw |= (1 << BIO_RW);
 
-       rq->bio = rq->biotail = bio;
        blk_rq_bio_prep(q, rq, bio);
-
        rq->buffer = rq->data = NULL;
-       rq->data_len = len;
        return 0;
 }
 
@@ -2569,10 +2630,9 @@ int blk_execute_rq(request_queue_t *q, struct gendisk *bd_disk,
                rq->sense_len = 0;
        }
 
-       rq->waiting = &wait;
+       rq->end_io_data = &wait;
        blk_execute_rq_nowait(q, bd_disk, rq, at_head, blk_end_sync_rq);
        wait_for_completion(&wait);
-       rq->waiting = NULL;
 
        if (rq->errors)
                err = -EIO;
@@ -2681,8 +2741,6 @@ EXPORT_SYMBOL_GPL(disk_round_stats);
  */
 void __blk_put_request(request_queue_t *q, struct request *req)
 {
-       struct request_list *rl = req->rl;
-
        if (unlikely(!q))
                return;
        if (unlikely(--req->ref_count))
@@ -2690,14 +2748,11 @@ void __blk_put_request(request_queue_t *q, struct request *req)
 
        elv_completed_request(q, req);
 
-       req->rq_status = RQ_INACTIVE;
-       req->rl = NULL;
-
        /*
         * Request may not have originated from ll_rw_blk. if not,
         * it didn't come out of our reserved rq pools
         */
-       if (rl) {
+       if (req->cmd_flags & REQ_ALLOCED) {
                int rw = rq_data_dir(req);
                int priv = req->cmd_flags & REQ_ELVPRIV;
 
@@ -2736,9 +2791,9 @@ EXPORT_SYMBOL(blk_put_request);
  */
 void blk_end_sync_rq(struct request *rq, int error)
 {
-       struct completion *waiting = rq->waiting;
+       struct completion *waiting = rq->end_io_data;
 
-       rq->waiting = NULL;
+       rq->end_io_data = NULL;
        __blk_put_request(rq->q, rq);
 
        /*
@@ -2749,41 +2804,6 @@ void blk_end_sync_rq(struct request *rq, int error)
 }
 EXPORT_SYMBOL(blk_end_sync_rq);
 
-/**
- * blk_congestion_wait - wait for a queue to become uncongested
- * @rw: READ or WRITE
- * @timeout: timeout in jiffies
- *
- * Waits for up to @timeout jiffies for a queue (any queue) to exit congestion.
- * If no queues are congested then just wait for the next request to be
- * returned.
- */
-long blk_congestion_wait(int rw, long timeout)
-{
-       long ret;
-       DEFINE_WAIT(wait);
-       wait_queue_head_t *wqh = &congestion_wqh[rw];
-
-       prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
-       ret = io_schedule_timeout(timeout);
-       finish_wait(wqh, &wait);
-       return ret;
-}
-
-EXPORT_SYMBOL(blk_congestion_wait);
-
-/**
- * blk_congestion_end - wake up sleepers on a congestion queue
- * @rw: READ or WRITE
- */
-void blk_congestion_end(int rw)
-{
-       wait_queue_head_t *wqh = &congestion_wqh[rw];
-
-       if (waitqueue_active(wqh))
-               wake_up(wqh);
-}
-
 /*
  * Has to be called with the request spinlock acquired
  */
@@ -2801,7 +2821,7 @@ static int attempt_merge(request_queue_t *q, struct request *req,
 
        if (rq_data_dir(req) != rq_data_dir(next)
            || req->rq_disk != next->rq_disk
-           || next->waiting || next->special)
+           || next->special)
                return 0;
 
        /*
@@ -2878,6 +2898,8 @@ static void init_request_from_bio(struct request *req, struct bio *bio)
 
        if (bio_sync(bio))
                req->cmd_flags |= REQ_RW_SYNC;
+       if (bio_rw_meta(bio))
+               req->cmd_flags |= REQ_RW_META;
 
        req->errors = 0;
        req->hard_sector = req->sector = bio->bi_sector;
@@ -2886,7 +2908,6 @@ static void init_request_from_bio(struct request *req, struct bio *bio)
        req->nr_phys_segments = bio_phys_segments(req->q, bio);
        req->nr_hw_segments = bio_hw_segments(req->q, bio);
        req->buffer = bio_data(bio);    /* see ->buffer comment above */
-       req->waiting = NULL;
        req->bio = req->biotail = bio;
        req->ioprio = bio_prio(bio);
        req->rq_disk = bio->bi_bdev->bd_disk;
@@ -2896,17 +2917,11 @@ static void init_request_from_bio(struct request *req, struct bio *bio)
 static int __make_request(request_queue_t *q, struct bio *bio)
 {
        struct request *req;
-       int el_ret, rw, nr_sectors, cur_nr_sectors, barrier, err, sync;
-       unsigned short prio;
-       sector_t sector;
+       int el_ret, nr_sectors, barrier, err;
+       const unsigned short prio = bio_prio(bio);
+       const int sync = bio_sync(bio);
 
-       sector = bio->bi_sector;
        nr_sectors = bio_sectors(bio);
-       cur_nr_sectors = bio_cur_sectors(bio);
-       prio = bio_prio(bio);
-
-       rw = bio_data_dir(bio);
-       sync = bio_sync(bio);
 
        /*
         * low level driver can indicate that it wants pages above a
@@ -2915,8 +2930,6 @@ static int __make_request(request_queue_t *q, struct bio *bio)
         */
        blk_queue_bounce(q, &bio);
 
-       spin_lock_prefetch(q->queue_lock);
-
        barrier = bio_barrier(bio);
        if (unlikely(barrier) && (q->next_ordered == QUEUE_ORDERED_NONE)) {
                err = -EOPNOTSUPP;
@@ -2964,9 +2977,9 @@ static int __make_request(request_queue_t *q, struct bio *bio)
                         * not touch req->buffer either...
                         */
                        req->buffer = bio_data(bio);
-                       req->current_nr_sectors = cur_nr_sectors;
-                       req->hard_cur_sectors = cur_nr_sectors;
-                       req->sector = req->hard_sector = sector;
+                       req->current_nr_sectors = bio_cur_sectors(bio);
+                       req->hard_cur_sectors = req->current_nr_sectors;
+                       req->sector = req->hard_sector = bio->bi_sector;
                        req->nr_sectors = req->hard_nr_sectors += nr_sectors;
                        req->ioprio = ioprio_best(req->ioprio, prio);
                        drive_stat_acct(req, nr_sectors, 0);
@@ -2984,7 +2997,7 @@ get_rq:
         * Grab a free request. This is might sleep but can not fail.
         * Returns with the queue unlocked.
         */
-       req = get_request_wait(q, rw, bio);
+       req = get_request_wait(q, bio_data_dir(bio), bio);
 
        /*
         * After dropping the lock and possibly sleeping here, our request
@@ -3071,6 +3084,7 @@ void generic_make_request(struct bio *bio)
 {
        request_queue_t *q;
        sector_t maxsector;
+       sector_t old_sector;
        int ret, nr_sectors = bio_sectors(bio);
        dev_t old_dev;
 
@@ -3099,7 +3113,7 @@ void generic_make_request(struct bio *bio)
         * NOTE: we don't repeat the blk_size check for each new device.
         * Stacking drivers are expected to know what they are doing.
         */
-       maxsector = -1;
+       old_sector = -1;
        old_dev = 0;
        do {
                char b[BDEVNAME_SIZE];
@@ -3133,15 +3147,31 @@ end_io:
                 */
                blk_partition_remap(bio);
 
-               if (maxsector != -1)
+               if (old_sector != -1)
                        blk_add_trace_remap(q, bio, old_dev, bio->bi_sector, 
-                                           maxsector);
+                                           old_sector);
 
                blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
 
-               maxsector = bio->bi_sector;
+               old_sector = bio->bi_sector;
                old_dev = bio->bi_bdev->bd_dev;
 
+               maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
+               if (maxsector) {
+                       sector_t sector = bio->bi_sector;
+
+                       if (maxsector < nr_sectors ||
+                                       maxsector - nr_sectors < sector) {
+                               /*
+                                * This may well happen - partitions are not
+                                * checked to make sure they are within the size
+                                * of the whole device.
+                                */
+                               handle_bad_sector(bio);
+                               goto end_io;
+                       }
+               }
+
                ret = q->make_request_fn(q, bio);
        } while (ret);
 }
@@ -3550,6 +3580,7 @@ void blk_rq_bio_prep(request_queue_t *q, struct request *rq, struct bio *bio)
        rq->hard_cur_sectors = rq->current_nr_sectors;
        rq->hard_nr_sectors = rq->nr_sectors = bio_sectors(bio);
        rq->buffer = bio_data(bio);
+       rq->data_len = bio->bi_size;
 
        rq->bio = rq->biotail = bio;
 }
@@ -3630,25 +3661,22 @@ EXPORT_SYMBOL(put_io_context);
 /* Called by the exitting task */
 void exit_io_context(void)
 {
-       unsigned long flags;
        struct io_context *ioc;
        struct cfq_io_context *cic;
 
-       local_irq_save(flags);
        task_lock(current);
        ioc = current->io_context;
        current->io_context = NULL;
-       ioc->task = NULL;
        task_unlock(current);
-       local_irq_restore(flags);
 
+       ioc->task = NULL;
        if (ioc->aic && ioc->aic->exit)
                ioc->aic->exit(ioc->aic);
        if (ioc->cic_root.rb_node != NULL) {
                cic = rb_entry(rb_first(&ioc->cic_root), struct cfq_io_context, rb_node);
                cic->exit(ioc);
        }
+
        put_io_context(ioc);
 }
 
@@ -3660,7 +3688,7 @@ void exit_io_context(void)
  * but since the current task itself holds a reference, the context can be
  * used in general code, so long as it stays within `current` context.
  */
-struct io_context *current_io_context(gfp_t gfp_flags)
+static struct io_context *current_io_context(gfp_t gfp_flags, int node)
 {
        struct task_struct *tsk = current;
        struct io_context *ret;
@@ -3669,11 +3697,11 @@ struct io_context *current_io_context(gfp_t gfp_flags)
        if (likely(ret))
                return ret;
 
-       ret = kmem_cache_alloc(iocontext_cachep, gfp_flags);
+       ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
        if (ret) {
                atomic_set(&ret->refcount, 1);
                ret->task = current;
-               ret->set_ioprio = NULL;
+               ret->ioprio_changed = 0;
                ret->last_waited = jiffies; /* doesn't matter... */
                ret->nr_batch_requests = 0; /* because this is 0 */
                ret->aic = NULL;
@@ -3693,10 +3721,10 @@ EXPORT_SYMBOL(current_io_context);
  *
  * This is always called in the context of the task which submitted the I/O.
  */
-struct io_context *get_io_context(gfp_t gfp_flags)
+struct io_context *get_io_context(gfp_t gfp_flags, int node)
 {
        struct io_context *ret;
-       ret = current_io_context(gfp_flags);
+       ret = current_io_context(gfp_flags, node);
        if (likely(ret))
                atomic_inc(&ret->refcount);
        return ret;
@@ -3769,14 +3797,14 @@ queue_requests_store(struct request_queue *q, const char *page, size_t count)
        blk_queue_congestion_threshold(q);
 
        if (rl->count[READ] >= queue_congestion_on_threshold(q))
-               set_queue_congested(q, READ);
+               blk_set_queue_congested(q, READ);
        else if (rl->count[READ] < queue_congestion_off_threshold(q))
-               clear_queue_congested(q, READ);
+               blk_clear_queue_congested(q, READ);
 
        if (rl->count[WRITE] >= queue_congestion_on_threshold(q))
-               set_queue_congested(q, WRITE);
+               blk_set_queue_congested(q, WRITE);
        else if (rl->count[WRITE] < queue_congestion_off_threshold(q))
-               clear_queue_congested(q, WRITE);
+               blk_clear_queue_congested(q, WRITE);
 
        if (rl->count[READ] >= q->nr_requests) {
                blk_set_queue_full(q, READ);
@@ -3809,9 +3837,6 @@ queue_ra_store(struct request_queue *q, const char *page, size_t count)
        ssize_t ret = queue_var_store(&ra_kb, page, count);
 
        spin_lock_irq(q->queue_lock);
-       if (ra_kb > (q->max_sectors >> 1))
-               ra_kb = (q->max_sectors >> 1);
-
        q->backing_dev_info.ra_pages = ra_kb >> (PAGE_CACHE_SHIFT - 10);
        spin_unlock_irq(q->queue_lock);