struct cfq_rb_root *service_tree;
struct cfq_queue *new_cfqq;
struct cfq_group *cfqg;
+ struct cfq_group *orig_cfqg;
/* Sectors dispatched in current dispatch round */
unsigned long nr_sectors;
};
unsigned int cfq_slice_async_rq;
unsigned int cfq_slice_idle;
unsigned int cfq_latency;
+ unsigned int cfq_group_isolation;
struct list_head cic_list;
/* List of cfq groups being managed on this device*/
struct hlist_head cfqg_list;
+ struct rcu_head rcu;
};
static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd);
unsigned int major, minor;
/* Do we need to take this reference */
- if (!css_tryget(&blkcg->css))
+ if (!blkiocg_css_tryget(blkcg))
return NULL;;
cfqg = cfqg_of_blkg(blkiocg_lookup_group(blkcg, key));
hlist_add_head(&cfqg->cfqd_node, &cfqd->cfqg_list);
done:
- css_put(&blkcg->css);
+ blkiocg_css_put(blkcg);
return cfqg;
}
struct cfq_rb_root *service_tree;
int left;
int new_cfqq = 1;
+ int group_changed = 0;
+
+#ifdef CONFIG_CFQ_GROUP_IOSCHED
+ if (!cfqd->cfq_group_isolation
+ && cfqq_type(cfqq) == SYNC_NOIDLE_WORKLOAD
+ && cfqq->cfqg && cfqq->cfqg != &cfqd->root_group) {
+ /* Move this cfq to root group */
+ cfq_log_cfqq(cfqd, cfqq, "moving to root group");
+ if (!RB_EMPTY_NODE(&cfqq->rb_node))
+ cfq_group_service_tree_del(cfqd, cfqq->cfqg);
+ cfqq->orig_cfqg = cfqq->cfqg;
+ cfqq->cfqg = &cfqd->root_group;
+ atomic_inc(&cfqd->root_group.ref);
+ group_changed = 1;
+ } else if (!cfqd->cfq_group_isolation
+ && cfqq_type(cfqq) == SYNC_WORKLOAD && cfqq->orig_cfqg) {
+ /* cfqq is sequential now needs to go to its original group */
+ BUG_ON(cfqq->cfqg != &cfqd->root_group);
+ if (!RB_EMPTY_NODE(&cfqq->rb_node))
+ cfq_group_service_tree_del(cfqd, cfqq->cfqg);
+ cfq_put_cfqg(cfqq->cfqg);
+ cfqq->cfqg = cfqq->orig_cfqg;
+ cfqq->orig_cfqg = NULL;
+ group_changed = 1;
+ cfq_log_cfqq(cfqd, cfqq, "moved to origin group");
+ }
+#endif
service_tree = service_tree_for(cfqq->cfqg, cfqq_prio(cfqq),
cfqq_type(cfqq), cfqd);
rb_link_node(&cfqq->rb_node, parent, p);
rb_insert_color(&cfqq->rb_node, &service_tree->rb);
service_tree->count++;
- if (add_front || !new_cfqq)
+ if ((add_front || !new_cfqq) && !group_changed)
return;
cfq_group_service_tree_add(cfqd, cfqq->cfqg);
}
return false;
/* We do for queues that were marked with idle window flag. */
- if (cfq_cfqq_idle_window(cfqq))
+ if (cfq_cfqq_idle_window(cfqq) &&
+ !(blk_queue_nonrot(cfqd->queue) && cfqd->hw_tag))
return true;
/*
static void cfq_put_queue(struct cfq_queue *cfqq)
{
struct cfq_data *cfqd = cfqq->cfqd;
- struct cfq_group *cfqg;
+ struct cfq_group *cfqg, *orig_cfqg;
BUG_ON(atomic_read(&cfqq->ref) <= 0);
BUG_ON(rb_first(&cfqq->sort_list));
BUG_ON(cfqq->allocated[READ] + cfqq->allocated[WRITE]);
cfqg = cfqq->cfqg;
+ orig_cfqg = cfqq->orig_cfqg;
if (unlikely(cfqd->active_queue == cfqq)) {
__cfq_slice_expired(cfqd, cfqq, 0);
BUG_ON(cfq_cfqq_on_rr(cfqq));
kmem_cache_free(cfq_pool, cfqq);
cfq_put_cfqg(cfqg);
+ if (orig_cfqg)
+ cfq_put_cfqg(orig_cfqg);
}
/*
* only if we processed at least one !rq_noidle request
*/
if (cfqd->serving_type == SYNC_WORKLOAD
- || cfqd->noidle_tree_requires_idle)
+ || cfqd->noidle_tree_requires_idle
+ || cfqq->cfqg->nr_cfqq == 1)
cfq_arm_slice_timer(cfqd);
}
}
cfq_put_queue(cfqd->async_idle_cfqq);
}
+static void cfq_cfqd_free(struct rcu_head *head)
+{
+ kfree(container_of(head, struct cfq_data, rcu));
+}
+
static void cfq_exit_queue(struct elevator_queue *e)
{
struct cfq_data *cfqd = e->elevator_data;
cfq_shutdown_timer_wq(cfqd);
/* Wait for cfqg->blkg->key accessors to exit their grace periods. */
- synchronize_rcu();
- kfree(cfqd);
+ call_rcu(&cfqd->rcu, cfq_cfqd_free);
}
static void *cfq_init_queue(struct request_queue *q)
cfqd->cfq_slice_async_rq = cfq_slice_async_rq;
cfqd->cfq_slice_idle = cfq_slice_idle;
cfqd->cfq_latency = 1;
+ cfqd->cfq_group_isolation = 0;
cfqd->hw_tag = -1;
cfqd->last_end_sync_rq = jiffies;
+ INIT_RCU_HEAD(&cfqd->rcu);
return cfqd;
}
SHOW_FUNCTION(cfq_slice_async_show, cfqd->cfq_slice[0], 1);
SHOW_FUNCTION(cfq_slice_async_rq_show, cfqd->cfq_slice_async_rq, 0);
SHOW_FUNCTION(cfq_low_latency_show, cfqd->cfq_latency, 0);
+SHOW_FUNCTION(cfq_group_isolation_show, cfqd->cfq_group_isolation, 0);
#undef SHOW_FUNCTION
#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \
STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1,
UINT_MAX, 0);
STORE_FUNCTION(cfq_low_latency_store, &cfqd->cfq_latency, 0, 1, 0);
+STORE_FUNCTION(cfq_group_isolation_store, &cfqd->cfq_group_isolation, 0, 1, 0);
#undef STORE_FUNCTION
#define CFQ_ATTR(name) \
CFQ_ATTR(slice_async_rq),
CFQ_ATTR(slice_idle),
CFQ_ATTR(low_latency),
+ CFQ_ATTR(group_isolation),
__ATTR_NULL
};
.elevator_owner = THIS_MODULE,
};
+#ifdef CONFIG_CFQ_GROUP_IOSCHED
+static struct blkio_policy_type blkio_policy_cfq = {
+ .ops = {
+ .blkio_unlink_group_fn = cfq_unlink_blkio_group,
+ .blkio_update_group_weight_fn = cfq_update_blkio_group_weight,
+ },
+};
+#else
+static struct blkio_policy_type blkio_policy_cfq;
+#endif
+
static int __init cfq_init(void)
{
/*
return -ENOMEM;
elv_register(&iosched_cfq);
+ blkio_policy_register(&blkio_policy_cfq);
return 0;
}
static void __exit cfq_exit(void)
{
DECLARE_COMPLETION_ONSTACK(all_gone);
+ blkio_policy_unregister(&blkio_policy_cfq);
elv_unregister(&iosched_cfq);
ioc_gone = &all_gone;
/* ioc_gone's update must be visible before reading ioc_count */