if (!cfq_cfqq_on_rr(cfqq))
cfq_add_cfqq_rr(cfqd, cfqq);
+
+ /*
+ * check if this request is a better next-serve candidate
+ */
+ cfqq->next_rq = cfq_choose_req(cfqd, cfqq->next_rq, rq);
+ BUG_ON(!cfqq->next_rq);
}
static inline void
*/
static void
__cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
- int preempted)
+ int preempted, int timed_out)
{
if (cfq_cfqq_wait_request(cfqq))
del_timer(&cfqd->idle_slice_timer);
- if (!preempted && !cfq_cfqq_dispatched(cfqq))
- cfq_schedule_dispatch(cfqd);
-
cfq_clear_cfqq_must_dispatch(cfqq);
cfq_clear_cfqq_wait_request(cfqq);
cfq_clear_cfqq_queue_new(cfqq);
* store what was left of this slice, if the queue idled out
* or was preempted
*/
- if (!cfq_cfqq_slice_new(cfqq))
+ if (timed_out && !cfq_cfqq_slice_new(cfqq))
cfqq->slice_resid = cfqq->slice_end - jiffies;
cfq_resort_rr_list(cfqq, preempted);
cfqd->dispatch_slice = 0;
}
-static inline void cfq_slice_expired(struct cfq_data *cfqd, int preempted)
+static inline void cfq_slice_expired(struct cfq_data *cfqd, int preempted,
+ int timed_out)
{
struct cfq_queue *cfqq = cfqd->active_queue;
if (cfqq)
- __cfq_slice_expired(cfqd, cfqq, preempted);
+ __cfq_slice_expired(cfqd, cfqq, preempted, timed_out);
}
/*
#define CIC_SEEKY(cic) ((cic)->seek_mean > (128 * 1024))
-static int cfq_arm_slice_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)
-
+static int cfq_arm_slice_timer(struct cfq_data *cfqd)
{
+ struct cfq_queue *cfqq = cfqd->active_queue;
struct cfq_io_context *cic;
unsigned long sl;
WARN_ON(!RB_EMPTY_ROOT(&cfqq->sort_list));
- WARN_ON(cfqq != cfqd->active_queue);
/*
* idle is disabled, either manually or by past process history
if (cfq_cfqq_fifo_expire(cfqq))
return NULL;
+
+ cfq_mark_cfqq_fifo_expire(cfqq);
+
if (list_empty(&cfqq->fifo))
return NULL;
fifo = cfq_cfqq_class_sync(cfqq);
rq = rq_entry_fifo(cfqq->fifo.next);
- if (time_after(jiffies, rq->start_time + cfqd->cfq_fifo_expire[fifo])) {
- cfq_mark_cfqq_fifo_expire(cfqq);
+ if (time_after(jiffies, rq->start_time + cfqd->cfq_fifo_expire[fifo]))
return rq;
- }
return NULL;
}
cfqq = NULL;
goto keep_queue;
} else if (cfq_cfqq_class_sync(cfqq)) {
- if (cfq_arm_slice_timer(cfqd, cfqq))
+ if (cfq_arm_slice_timer(cfqd))
return NULL;
}
expire:
- cfq_slice_expired(cfqd, 0);
+ cfq_slice_expired(cfqd, 0, 0);
new_queue:
cfqq = cfq_set_active_queue(cfqd);
keep_queue:
* expire an async queue immediately if it has used up its slice. idle
* queue always expire after 1 dispatch round.
*/
- if ((!cfq_cfqq_sync(cfqq) &&
+ if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) &&
cfqd->dispatch_slice >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
- cfq_class_idle(cfqq)) {
+ cfq_class_idle(cfqq))) {
cfqq->slice_end = jiffies + 1;
- cfq_slice_expired(cfqd, 0);
+ cfq_slice_expired(cfqd, 0, 0);
}
return dispatched;
dispatched += cfq_forced_dispatch_cfqqs(&cfqd->cur_rr);
dispatched += cfq_forced_dispatch_cfqqs(&cfqd->idle_rr);
- cfq_slice_expired(cfqd, 0);
+ cfq_slice_expired(cfqd, 0, 0);
BUG_ON(cfqd->busy_queues);
while ((cfqq = cfq_select_queue(cfqd)) != NULL) {
int max_dispatch;
- /*
- * Don't repeat dispatch from the previous queue.
- */
- if (prev_cfqq == cfqq)
- break;
+ if (cfqd->busy_queues > 1) {
+ /*
+ * Don't repeat dispatch from the previous queue.
+ */
+ if (prev_cfqq == cfqq)
+ break;
+
+ /*
+ * So we have dispatched before in this round, if the
+ * next queue has idling enabled (must be sync), don't
+ * allow it service until the previous have continued.
+ */
+ if (cfqd->rq_in_driver && cfq_cfqq_idle_window(cfqq))
+ break;
+ }
cfq_clear_cfqq_must_dispatch(cfqq);
cfq_clear_cfqq_wait_request(cfqq);
max_dispatch = 1;
dispatched += __cfq_dispatch_requests(cfqd, cfqq, max_dispatch);
-
- /*
- * If the dispatch cfqq has idling enabled and is still
- * the active queue, break out.
- */
- if (cfq_cfqq_idle_window(cfqq) && cfqd->active_queue)
- break;
-
prev_cfqq = cfqq;
}
BUG_ON(cfqq->allocated[READ] + cfqq->allocated[WRITE]);
BUG_ON(cfq_cfqq_on_rr(cfqq));
- if (unlikely(cfqd->active_queue == cfqq))
- __cfq_slice_expired(cfqd, cfqq, 0);
+ if (unlikely(cfqd->active_queue == cfqq)) {
+ __cfq_slice_expired(cfqd, cfqq, 0, 0);
+ cfq_schedule_dispatch(cfqd);
+ }
/*
* it's on the empty list and still hashed
static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{
- if (unlikely(cfqq == cfqd->active_queue))
- __cfq_slice_expired(cfqd, cfqq, 0);
+ if (unlikely(cfqq == cfqd->active_queue)) {
+ __cfq_slice_expired(cfqd, cfqq, 0, 0);
+ cfq_schedule_dispatch(cfqd);
+ }
cfq_put_queue(cfqq);
}
atomic_set(&cfqq->ref, 0);
cfqq->cfqd = cfqd;
- cfq_mark_cfqq_idle_window(cfqq);
+ if (key != CFQ_KEY_ASYNC)
+ cfq_mark_cfqq_idle_window(cfqq);
+
cfq_mark_cfqq_prio_changed(cfqq);
cfq_mark_cfqq_queue_new(cfqq);
cfq_init_prio_data(cfqq);
cfq_clear_cfqq_idle_window(cfqq);
}
-
/*
* Check if new_cfqq should preempt the currently active queue. Return 0 for
* no or if we aren't sure, a 1 will cause a preempt.
*/
static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{
- cfq_slice_expired(cfqd, 1);
+ cfq_slice_expired(cfqd, 1, 1);
/*
* Put the new queue at the front of the of the current list,
if (rq_is_meta(rq))
cfqq->meta_pending++;
- /*
- * check if this request is a better next-serve candidate)) {
- */
- cfqq->next_rq = cfq_choose_req(cfqd, cfqq->next_rq, rq);
- BUG_ON(!cfqq->next_rq);
-
/*
* we never wait for an async request and we don't allow preemption
* of an async request. so just return early
*/
if (cic == cfqd->active_cic &&
del_timer(&cfqd->idle_slice_timer)) {
- cfq_slice_expired(cfqd, 0);
+ cfq_slice_expired(cfqd, 0, 0);
blk_start_queueing(cfqd->queue);
}
return;
cfq_clear_cfqq_slice_new(cfqq);
}
if (cfq_slice_used(cfqq))
- cfq_slice_expired(cfqd, 0);
+ cfq_slice_expired(cfqd, 0, 1);
else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list)) {
- if (!cfq_arm_slice_timer(cfqd, cfqq))
+ if (!cfq_arm_slice_timer(cfqd))
cfq_schedule_dispatch(cfqd);
}
}
struct cfq_data *cfqd = (struct cfq_data *) data;
struct cfq_queue *cfqq;
unsigned long flags;
+ int timed_out = 1;
spin_lock_irqsave(cfqd->queue->queue_lock, flags);
if ((cfqq = cfqd->active_queue) != NULL) {
+ timed_out = 0;
+
/*
* expired
*/
}
}
expire:
- cfq_slice_expired(cfqd, 0);
+ cfq_slice_expired(cfqd, 0, timed_out);
out_kick:
cfq_schedule_dispatch(cfqd);
out_cont:
spin_lock_irq(q->queue_lock);
if (cfqd->active_queue)
- __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
+ __cfq_slice_expired(cfqd, cfqd->active_queue, 0, 0);
while (!list_empty(&cfqd->cic_list)) {
struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,