*
* Copyright (C) 2003 Jens Axboe <axboe@suse.de>
*/
-#include <linux/kernel.h>
-#include <linux/fs.h>
-#include <linux/blkdev.h>
-#include <linux/elevator.h>
-#include <linux/bio.h>
#include <linux/config.h>
#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/compiler.h>
+#include <linux/blkdev.h>
+#include <linux/elevator.h>
#include <linux/hash.h>
#include <linux/rbtree.h>
-#include <linux/mempool.h>
#include <linux/ioprio.h>
-#include <linux/writeback.h>
/*
* tunables
*/
-static int cfq_quantum = 4; /* max queue in one round of service */
-static int cfq_queued = 8; /* minimum rq allocate limit per-queue*/
-static int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 };
-static int cfq_back_max = 16 * 1024; /* maximum backwards seek, in KiB */
-static int cfq_back_penalty = 2; /* penalty of a backwards seek */
+static const int cfq_quantum = 4; /* max queue in one round of service */
+static const int cfq_queued = 8; /* minimum rq allocate limit per-queue*/
+static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 };
+static const int cfq_back_max = 16 * 1024; /* maximum backwards seek, in KiB */
+static const int cfq_back_penalty = 2; /* penalty of a backwards seek */
-static int cfq_slice_sync = HZ / 10;
+static const int cfq_slice_sync = HZ / 10;
static int cfq_slice_async = HZ / 25;
-static int cfq_slice_async_rq = 2;
+static const int cfq_slice_async_rq = 2;
static int cfq_slice_idle = HZ / 100;
#define CFQ_IDLE_GRACE (HZ / 10)
/*
* disable queueing at the driver/hardware level
*/
-static int cfq_max_depth = 2;
+static const int cfq_max_depth = 2;
+
+static DEFINE_RWLOCK(cfq_exit_lock);
/*
* for the hash of cfqq inside the cfqd
static kmem_cache_t *cfq_pool;
static kmem_cache_t *cfq_ioc_pool;
+static atomic_t ioc_count = ATOMIC_INIT(0);
+static struct completion *ioc_gone;
+
#define CFQ_PRIO_LISTS IOPRIO_BE_NR
#define cfq_class_idle(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE)
#define cfq_class_be(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_BE)
* Per block device queue structure
*/
struct cfq_data {
- atomic_t ref;
request_queue_t *queue;
/*
unsigned int cfq_slice_async_rq;
unsigned int cfq_slice_idle;
unsigned int cfq_max_depth;
+
+ struct list_head cic_list;
};
/*
CFQ_CFQQ_FLAG_fifo_expire,
CFQ_CFQQ_FLAG_idle_window,
CFQ_CFQQ_FLAG_prio_changed,
- CFQ_CFQQ_FLAG_expired,
};
#define CFQ_CFQQ_FNS(name) \
CFQ_CFQQ_FNS(fifo_expire);
CFQ_CFQQ_FNS(idle_window);
CFQ_CFQQ_FNS(prio_changed);
-CFQ_CFQQ_FNS(expired);
#undef CFQ_CFQQ_FNS
enum cfq_rq_state_flags {
static struct cfq_queue *cfq_find_cfq_hash(struct cfq_data *, unsigned int, unsigned short);
static void cfq_dispatch_insert(request_queue_t *, struct cfq_rq *);
-static void cfq_put_cfqd(struct cfq_data *cfqd);
+static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, unsigned int key, struct task_struct *tsk, gfp_t gfp_mask);
#define process_sync(tsk) ((tsk)->flags & PF_SYNCWRITE)
*/
static inline void cfq_schedule_dispatch(struct cfq_data *cfqd)
{
- if (!cfqd->rq_in_driver && cfqd->busy_queues)
+ if (cfqd->busy_queues)
kblockd_schedule_work(&cfqd->unplug_work);
}
cfqq->slice_left = 0;
cfq_clear_cfqq_must_alloc_slice(cfqq);
cfq_clear_cfqq_fifo_expire(cfqq);
- cfq_clear_cfqq_expired(cfqq);
}
cfqd->active_queue = cfqq;
}
+/*
+ * current cfqq expired its slice (or was too idle), select new one
+ */
+static void
+__cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
+ int preempted)
+{
+ unsigned long now = jiffies;
+
+ if (cfq_cfqq_wait_request(cfqq))
+ del_timer(&cfqd->idle_slice_timer);
+
+ if (!preempted && !cfq_cfqq_dispatched(cfqq)) {
+ cfqq->service_last = now;
+ cfq_schedule_dispatch(cfqd);
+ }
+
+ cfq_clear_cfqq_must_dispatch(cfqq);
+ cfq_clear_cfqq_wait_request(cfqq);
+
+ /*
+ * store what was left of this slice, if the queue idled out
+ * or was preempted
+ */
+ if (time_after(cfqq->slice_end, now))
+ cfqq->slice_left = cfqq->slice_end - now;
+ else
+ cfqq->slice_left = 0;
+
+ if (cfq_cfqq_on_rr(cfqq))
+ cfq_resort_rr_list(cfqq, preempted);
+
+ if (cfqq == cfqd->active_queue)
+ cfqd->active_queue = NULL;
+
+ if (cfqd->active_cic) {
+ put_io_context(cfqd->active_cic->ioc);
+ cfqd->active_cic = NULL;
+ }
+
+ cfqd->dispatch_slice = 0;
+}
+
+static inline void cfq_slice_expired(struct cfq_data *cfqd, int preempted)
+{
+ struct cfq_queue *cfqq = cfqd->active_queue;
+
+ if (cfqq)
+ __cfq_slice_expired(cfqd, cfqq, preempted);
+}
+
/*
* 0
* 0,1
static struct cfq_queue *cfq_set_active_queue(struct cfq_data *cfqd)
{
- struct cfq_queue *cfqq;
-
- /*
- * if current queue is expired but not done with its requests yet,
- * wait for that to happen
- */
- if ((cfqq = cfqd->active_queue) != NULL) {
- if (cfq_cfqq_expired(cfqq) && cfq_cfqq_dispatched(cfqq))
- return NULL;
- }
+ struct cfq_queue *cfqq = NULL;
/*
* if current list is non-empty, grab first entry. if it is empty,
return cfqq;
}
-/*
- * current cfqq expired its slice (or was too idle), select new one
- */
-static void
-__cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
- int preempted)
-{
- unsigned long now = jiffies;
-
- if (cfq_cfqq_wait_request(cfqq))
- del_timer(&cfqd->idle_slice_timer);
-
- if (!preempted && !cfq_cfqq_dispatched(cfqq))
- cfqq->service_last = now;
-
- cfq_clear_cfqq_must_dispatch(cfqq);
- cfq_clear_cfqq_wait_request(cfqq);
-
- /*
- * store what was left of this slice, if the queue idled out
- * or was preempted
- */
- if (time_after(cfqq->slice_end, now))
- cfqq->slice_left = cfqq->slice_end - now;
- else
- cfqq->slice_left = 0;
-
- if (cfq_cfqq_on_rr(cfqq))
- cfq_resort_rr_list(cfqq, preempted);
-
- if (cfqq == cfqd->active_queue)
- cfqd->active_queue = NULL;
-
- if (cfqd->active_cic) {
- put_io_context(cfqd->active_cic->ioc);
- cfqd->active_cic = NULL;
- }
-
- cfqd->dispatch_slice = 0;
-}
-
-static inline void cfq_slice_expired(struct cfq_data *cfqd, int preempted)
-{
- struct cfq_queue *cfqq = cfqd->active_queue;
-
- if (cfqq) {
- /*
- * use deferred expiry, if there are requests in progress as
- * not to disturb the slice of the next queue
- */
- if (cfq_cfqq_dispatched(cfqq))
- cfq_mark_cfqq_expired(cfqq);
- else
- __cfq_slice_expired(cfqd, cfqq, preempted);
- }
-}
-
static int cfq_arm_slice_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{
+ unsigned long sl;
+
WARN_ON(!RB_EMPTY(&cfqq->sort_list));
WARN_ON(cfqq != cfqd->active_queue);
cfq_mark_cfqq_must_dispatch(cfqq);
cfq_mark_cfqq_wait_request(cfqq);
- if (!timer_pending(&cfqd->idle_slice_timer)) {
- unsigned long slice_left = min(cfqq->slice_end - 1, (unsigned long) cfqd->cfq_slice_idle);
-
- cfqd->idle_slice_timer.expires = jiffies + slice_left;
- add_timer(&cfqd->idle_slice_timer);
- }
-
+ sl = min(cfqq->slice_end - 1, (unsigned long) cfqd->cfq_slice_idle);
+ mod_timer(&cfqd->idle_slice_timer, jiffies + sl);
return 1;
}
if (!cfqq)
goto new_queue;
- if (cfq_cfqq_expired(cfqq))
- goto new_queue;
-
/*
* slice has expired
*/
BUG_ON(cfqq->allocated[READ] + cfqq->allocated[WRITE]);
BUG_ON(cfq_cfqq_on_rr(cfqq));
- if (unlikely(cfqd->active_queue == cfqq)) {
+ if (unlikely(cfqd->active_queue == cfqq))
__cfq_slice_expired(cfqd, cfqq, 0);
- cfq_schedule_dispatch(cfqd);
- }
-
- cfq_put_cfqd(cfqq->cfqd);
/*
* it's on the empty list and still hashed
hlist_for_each_safe(entry, next, hash_list) {
struct cfq_queue *__cfqq = list_entry_qhash(entry);
- const unsigned short __p = IOPRIO_PRIO_VALUE(__cfqq->ioprio_class, __cfqq->ioprio);
+ const unsigned short __p = IOPRIO_PRIO_VALUE(__cfqq->org_ioprio_class, __cfqq->org_ioprio);
if (__cfqq->key == key && (__p == prio || prio == CFQ_KEY_ANY))
return __cfqq;
{
struct cfq_io_context *__cic;
struct list_head *entry, *next;
+ int freed = 1;
list_for_each_safe(entry, next, &cic->list) {
__cic = list_entry(entry, struct cfq_io_context, list);
kmem_cache_free(cfq_ioc_pool, __cic);
+ freed++;
}
kmem_cache_free(cfq_ioc_pool, cic);
+ if (atomic_sub_and_test(freed, &ioc_count) && ioc_gone)
+ complete(ioc_gone);
+}
+
+static void cfq_trim(struct io_context *ioc)
+{
+ ioc->set_ioprio = NULL;
+ if (ioc->cic)
+ cfq_free_io_context(ioc->cic);
}
/*
*/
static void cfq_exit_single_io_context(struct cfq_io_context *cic)
{
- struct cfq_data *cfqd = cic->cfqq->cfqd;
- request_queue_t *q = cfqd->queue;
+ struct cfq_data *cfqd = cic->key;
+ request_queue_t *q;
+
+ if (!cfqd)
+ return;
+
+ q = cfqd->queue;
WARN_ON(!irqs_disabled());
spin_lock(q->queue_lock);
- if (unlikely(cic->cfqq == cfqd->active_queue)) {
- __cfq_slice_expired(cfqd, cic->cfqq, 0);
- cfq_schedule_dispatch(cfqd);
+ if (cic->cfqq[ASYNC]) {
+ if (unlikely(cic->cfqq[ASYNC] == cfqd->active_queue))
+ __cfq_slice_expired(cfqd, cic->cfqq[ASYNC], 0);
+ cfq_put_queue(cic->cfqq[ASYNC]);
+ cic->cfqq[ASYNC] = NULL;
+ }
+
+ if (cic->cfqq[SYNC]) {
+ if (unlikely(cic->cfqq[SYNC] == cfqd->active_queue))
+ __cfq_slice_expired(cfqd, cic->cfqq[SYNC], 0);
+ cfq_put_queue(cic->cfqq[SYNC]);
+ cic->cfqq[SYNC] = NULL;
}
- cfq_put_queue(cic->cfqq);
- cic->cfqq = NULL;
+ cic->key = NULL;
+ list_del_init(&cic->queue_list);
spin_unlock(q->queue_lock);
}
-/*
- * Another task may update the task cic list, if it is doing a queue lookup
- * on its behalf. cfq_cic_lock excludes such concurrent updates
- */
static void cfq_exit_io_context(struct cfq_io_context *cic)
{
struct cfq_io_context *__cic;
/*
* put the reference this task is holding to the various queues
*/
+ read_lock(&cfq_exit_lock);
list_for_each(entry, &cic->list) {
__cic = list_entry(entry, struct cfq_io_context, list);
cfq_exit_single_io_context(__cic);
}
cfq_exit_single_io_context(cic);
+ read_unlock(&cfq_exit_lock);
local_irq_restore(flags);
}
if (cic) {
INIT_LIST_HEAD(&cic->list);
- cic->cfqq = NULL;
+ cic->cfqq[ASYNC] = NULL;
+ cic->cfqq[SYNC] = NULL;
cic->key = NULL;
cic->last_end_request = jiffies;
cic->ttime_total = 0;
cic->ttime_mean = 0;
cic->dtor = cfq_free_io_context;
cic->exit = cfq_exit_io_context;
+ INIT_LIST_HEAD(&cic->queue_list);
+ atomic_inc(&ioc_count);
}
return cic;
cfq_clear_cfqq_prio_changed(cfqq);
}
-static inline void changed_ioprio(struct cfq_queue *cfqq)
+static inline void changed_ioprio(struct cfq_io_context *cic)
{
- if (cfqq) {
- struct cfq_data *cfqd = cfqq->cfqd;
-
+ struct cfq_data *cfqd = cic->key;
+ struct cfq_queue *cfqq;
+ if (cfqd) {
spin_lock(cfqd->queue->queue_lock);
- cfq_mark_cfqq_prio_changed(cfqq);
- cfq_init_prio_data(cfqq);
+ cfqq = cic->cfqq[ASYNC];
+ if (cfqq) {
+ struct cfq_queue *new_cfqq;
+ new_cfqq = cfq_get_queue(cfqd, CFQ_KEY_ASYNC,
+ cic->ioc->task, GFP_ATOMIC);
+ if (new_cfqq) {
+ cic->cfqq[ASYNC] = new_cfqq;
+ cfq_put_queue(cfqq);
+ }
+ }
+ cfqq = cic->cfqq[SYNC];
+ if (cfqq) {
+ cfq_mark_cfqq_prio_changed(cfqq);
+ cfq_init_prio_data(cfqq);
+ }
spin_unlock(cfqd->queue->queue_lock);
}
}
*/
static int cfq_ioc_set_ioprio(struct io_context *ioc, unsigned int ioprio)
{
- struct cfq_io_context *cic = ioc->cic;
+ struct cfq_io_context *cic;
+
+ write_lock(&cfq_exit_lock);
- changed_ioprio(cic->cfqq);
+ cic = ioc->cic;
+
+ changed_ioprio(cic);
list_for_each_entry(cic, &cic->list, list)
- changed_ioprio(cic->cfqq);
+ changed_ioprio(cic);
+
+ write_unlock(&cfq_exit_lock);
return 0;
}
static struct cfq_queue *
-cfq_get_queue(struct cfq_data *cfqd, unsigned int key, unsigned short ioprio,
+cfq_get_queue(struct cfq_data *cfqd, unsigned int key, struct task_struct *tsk,
gfp_t gfp_mask)
{
const int hashval = hash_long(key, CFQ_QHASH_SHIFT);
struct cfq_queue *cfqq, *new_cfqq = NULL;
+ unsigned short ioprio;
retry:
+ ioprio = tsk->ioprio;
cfqq = __cfq_find_cfq_hash(cfqd, key, ioprio, hashval);
if (!cfqq) {
hlist_add_head(&cfqq->cfq_hash, &cfqd->cfq_hash[hashval]);
atomic_set(&cfqq->ref, 0);
cfqq->cfqd = cfqd;
- atomic_inc(&cfqd->ref);
cfqq->service_last = 0;
/*
* set ->slice_left to allow preemption for a new process
if (!ioc)
return NULL;
+restart:
if ((cic = ioc->cic) == NULL) {
cic = cfq_alloc_io_context(cfqd, gfp_mask);
* manually increment generic io_context usage count, it
* cannot go away since we are already holding one ref to it
*/
- ioc->cic = cic;
- ioc->set_ioprio = cfq_ioc_set_ioprio;
cic->ioc = ioc;
cic->key = cfqd;
- atomic_inc(&cfqd->ref);
+ read_lock(&cfq_exit_lock);
+ ioc->set_ioprio = cfq_ioc_set_ioprio;
+ ioc->cic = cic;
+ list_add(&cic->queue_list, &cfqd->cic_list);
+ read_unlock(&cfq_exit_lock);
} else {
struct cfq_io_context *__cic;
if (cic->key == cfqd)
goto out;
+ if (unlikely(!cic->key)) {
+ read_lock(&cfq_exit_lock);
+ if (list_empty(&cic->list))
+ ioc->cic = NULL;
+ else
+ ioc->cic = list_entry(cic->list.next,
+ struct cfq_io_context,
+ list);
+ read_unlock(&cfq_exit_lock);
+ kmem_cache_free(cfq_ioc_pool, cic);
+ atomic_dec(&ioc_count);
+ goto restart;
+ }
+
/*
* cic exists, check if we already are there. linear search
* should be ok here, the list will usually not be more than
cic = __cic;
goto out;
}
+ if (unlikely(!__cic->key)) {
+ read_lock(&cfq_exit_lock);
+ list_del(&__cic->list);
+ read_unlock(&cfq_exit_lock);
+ kmem_cache_free(cfq_ioc_pool, __cic);
+ atomic_dec(&ioc_count);
+ goto restart;
+ }
}
/*
__cic->ioc = ioc;
__cic->key = cfqd;
- atomic_inc(&cfqd->ref);
+ read_lock(&cfq_exit_lock);
list_add(&__cic->list, &cic->list);
+ list_add(&__cic->queue_list, &cfqd->cic_list);
+ read_unlock(&cfq_exit_lock);
cic = __cic;
}
cfqq->service_last = now;
cfq_resort_rr_list(cfqq, 0);
}
- if (cfq_cfqq_expired(cfqq)) {
- __cfq_slice_expired(cfqd, cfqq, 0);
- cfq_schedule_dispatch(cfqd);
- }
+ cfq_schedule_dispatch(cfqd);
}
if (cfq_crq_is_sync(crq))
struct cfq_queue *cfqq;
struct cfq_rq *crq;
unsigned long flags;
+ int is_sync = key != CFQ_KEY_ASYNC;
might_sleep_if(gfp_mask & __GFP_WAIT);
if (!cic)
goto queue_fail;
- if (!cic->cfqq) {
- cfqq = cfq_get_queue(cfqd, key, tsk->ioprio, gfp_mask);
+ if (!cic->cfqq[is_sync]) {
+ cfqq = cfq_get_queue(cfqd, key, tsk, gfp_mask);
if (!cfqq)
goto queue_fail;
- cic->cfqq = cfqq;
+ cic->cfqq[is_sync] = cfqq;
} else
- cfqq = cic->cfqq;
+ cfqq = cic->cfqq[is_sync];
cfqq->allocated[rw]++;
cfq_clear_cfqq_must_alloc(cfqq);
crq->cfq_queue = cfqq;
crq->io_context = cic;
- if (rw == READ || process_sync(tsk))
+ if (is_sync)
cfq_mark_crq_is_sync(crq);
else
cfq_clear_crq_is_sync(crq);
blk_sync_queue(cfqd->queue);
}
-static void cfq_put_cfqd(struct cfq_data *cfqd)
+static void cfq_exit_queue(elevator_t *e)
{
+ struct cfq_data *cfqd = e->elevator_data;
request_queue_t *q = cfqd->queue;
- if (!atomic_dec_and_test(&cfqd->ref))
- return;
+ cfq_shutdown_timer_wq(cfqd);
+ write_lock(&cfq_exit_lock);
+ spin_lock_irq(q->queue_lock);
+ if (cfqd->active_queue)
+ __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
+ while(!list_empty(&cfqd->cic_list)) {
+ struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,
+ struct cfq_io_context,
+ queue_list);
+ if (cic->cfqq[ASYNC]) {
+ cfq_put_queue(cic->cfqq[ASYNC]);
+ cic->cfqq[ASYNC] = NULL;
+ }
+ if (cic->cfqq[SYNC]) {
+ cfq_put_queue(cic->cfqq[SYNC]);
+ cic->cfqq[SYNC] = NULL;
+ }
+ cic->key = NULL;
+ list_del_init(&cic->queue_list);
+ }
+ spin_unlock_irq(q->queue_lock);
+ write_unlock(&cfq_exit_lock);
cfq_shutdown_timer_wq(cfqd);
- blk_put_queue(q);
mempool_destroy(cfqd->crq_pool);
kfree(cfqd->crq_hash);
kfree(cfqd);
}
-static void cfq_exit_queue(elevator_t *e)
-{
- struct cfq_data *cfqd = e->elevator_data;
-
- cfq_shutdown_timer_wq(cfqd);
- cfq_put_cfqd(cfqd);
-}
-
static int cfq_init_queue(request_queue_t *q, elevator_t *e)
{
struct cfq_data *cfqd;
INIT_LIST_HEAD(&cfqd->cur_rr);
INIT_LIST_HEAD(&cfqd->idle_rr);
INIT_LIST_HEAD(&cfqd->empty_list);
+ INIT_LIST_HEAD(&cfqd->cic_list);
cfqd->crq_hash = kmalloc(sizeof(struct hlist_head) * CFQ_MHASH_ENTRIES, GFP_KERNEL);
if (!cfqd->crq_hash)
e->elevator_data = cfqd;
cfqd->queue = q;
- atomic_inc(&q->refcnt);
cfqd->max_queued = q->nr_requests / 4;
q->nr_batching = cfq_queued;
INIT_WORK(&cfqd->unplug_work, cfq_kick_queue, q);
- atomic_set(&cfqd->ref, 1);
-
cfqd->cfq_queued = cfq_queued;
cfqd->cfq_quantum = cfq_quantum;
cfqd->cfq_fifo_expire[0] = cfq_fifo_expire[0];
/*
* sysfs parts below -->
*/
-struct cfq_fs_entry {
- struct attribute attr;
- ssize_t (*show)(struct cfq_data *, char *);
- ssize_t (*store)(struct cfq_data *, const char *, size_t);
-};
static ssize_t
cfq_var_show(unsigned int var, char *page)
}
#define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \
-static ssize_t __FUNC(struct cfq_data *cfqd, char *page) \
+static ssize_t __FUNC(elevator_t *e, char *page) \
{ \
+ struct cfq_data *cfqd = e->elevator_data; \
unsigned int __data = __VAR; \
if (__CONV) \
__data = jiffies_to_msecs(__data); \
#undef SHOW_FUNCTION
#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \
-static ssize_t __FUNC(struct cfq_data *cfqd, const char *page, size_t count) \
+static ssize_t __FUNC(elevator_t *e, const char *page, size_t count) \
{ \
+ struct cfq_data *cfqd = e->elevator_data; \
unsigned int __data; \
int ret = cfq_var_store(&__data, (page), count); \
if (__data < (MIN)) \
STORE_FUNCTION(cfq_max_depth_store, &cfqd->cfq_max_depth, 1, UINT_MAX, 0);
#undef STORE_FUNCTION
-static struct cfq_fs_entry cfq_quantum_entry = {
+static struct elv_fs_entry cfq_quantum_entry = {
.attr = {.name = "quantum", .mode = S_IRUGO | S_IWUSR },
.show = cfq_quantum_show,
.store = cfq_quantum_store,
};
-static struct cfq_fs_entry cfq_queued_entry = {
+static struct elv_fs_entry cfq_queued_entry = {
.attr = {.name = "queued", .mode = S_IRUGO | S_IWUSR },
.show = cfq_queued_show,
.store = cfq_queued_store,
};
-static struct cfq_fs_entry cfq_fifo_expire_sync_entry = {
+static struct elv_fs_entry cfq_fifo_expire_sync_entry = {
.attr = {.name = "fifo_expire_sync", .mode = S_IRUGO | S_IWUSR },
.show = cfq_fifo_expire_sync_show,
.store = cfq_fifo_expire_sync_store,
};
-static struct cfq_fs_entry cfq_fifo_expire_async_entry = {
+static struct elv_fs_entry cfq_fifo_expire_async_entry = {
.attr = {.name = "fifo_expire_async", .mode = S_IRUGO | S_IWUSR },
.show = cfq_fifo_expire_async_show,
.store = cfq_fifo_expire_async_store,
};
-static struct cfq_fs_entry cfq_back_max_entry = {
+static struct elv_fs_entry cfq_back_max_entry = {
.attr = {.name = "back_seek_max", .mode = S_IRUGO | S_IWUSR },
.show = cfq_back_max_show,
.store = cfq_back_max_store,
};
-static struct cfq_fs_entry cfq_back_penalty_entry = {
+static struct elv_fs_entry cfq_back_penalty_entry = {
.attr = {.name = "back_seek_penalty", .mode = S_IRUGO | S_IWUSR },
.show = cfq_back_penalty_show,
.store = cfq_back_penalty_store,
};
-static struct cfq_fs_entry cfq_slice_sync_entry = {
+static struct elv_fs_entry cfq_slice_sync_entry = {
.attr = {.name = "slice_sync", .mode = S_IRUGO | S_IWUSR },
.show = cfq_slice_sync_show,
.store = cfq_slice_sync_store,
};
-static struct cfq_fs_entry cfq_slice_async_entry = {
+static struct elv_fs_entry cfq_slice_async_entry = {
.attr = {.name = "slice_async", .mode = S_IRUGO | S_IWUSR },
.show = cfq_slice_async_show,
.store = cfq_slice_async_store,
};
-static struct cfq_fs_entry cfq_slice_async_rq_entry = {
+static struct elv_fs_entry cfq_slice_async_rq_entry = {
.attr = {.name = "slice_async_rq", .mode = S_IRUGO | S_IWUSR },
.show = cfq_slice_async_rq_show,
.store = cfq_slice_async_rq_store,
};
-static struct cfq_fs_entry cfq_slice_idle_entry = {
+static struct elv_fs_entry cfq_slice_idle_entry = {
.attr = {.name = "slice_idle", .mode = S_IRUGO | S_IWUSR },
.show = cfq_slice_idle_show,
.store = cfq_slice_idle_store,
};
-static struct cfq_fs_entry cfq_max_depth_entry = {
+static struct elv_fs_entry cfq_max_depth_entry = {
.attr = {.name = "max_depth", .mode = S_IRUGO | S_IWUSR },
.show = cfq_max_depth_show,
.store = cfq_max_depth_store,
};
-static struct attribute *default_attrs[] = {
+static struct attribute *cfq_attrs[] = {
&cfq_quantum_entry.attr,
&cfq_queued_entry.attr,
&cfq_fifo_expire_sync_entry.attr,
NULL,
};
-#define to_cfq(atr) container_of((atr), struct cfq_fs_entry, attr)
-
-static ssize_t
-cfq_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
-{
- elevator_t *e = container_of(kobj, elevator_t, kobj);
- struct cfq_fs_entry *entry = to_cfq(attr);
-
- if (!entry->show)
- return -EIO;
-
- return entry->show(e->elevator_data, page);
-}
-
-static ssize_t
-cfq_attr_store(struct kobject *kobj, struct attribute *attr,
- const char *page, size_t length)
-{
- elevator_t *e = container_of(kobj, elevator_t, kobj);
- struct cfq_fs_entry *entry = to_cfq(attr);
-
- if (!entry->store)
- return -EIO;
-
- return entry->store(e->elevator_data, page, length);
-}
-
-static struct sysfs_ops cfq_sysfs_ops = {
- .show = cfq_attr_show,
- .store = cfq_attr_store,
-};
-
-static struct kobj_type cfq_ktype = {
- .sysfs_ops = &cfq_sysfs_ops,
- .default_attrs = default_attrs,
-};
-
static struct elevator_type iosched_cfq = {
.ops = {
.elevator_merge_fn = cfq_merge,
.elevator_may_queue_fn = cfq_may_queue,
.elevator_init_fn = cfq_init_queue,
.elevator_exit_fn = cfq_exit_queue,
+ .trim = cfq_trim,
},
- .elevator_ktype = &cfq_ktype,
+ .elevator_attrs = cfq_attrs,
.elevator_name = "cfq",
.elevator_owner = THIS_MODULE,
};
static void __exit cfq_exit(void)
{
+ DECLARE_COMPLETION(all_gone);
elv_unregister(&iosched_cfq);
+ ioc_gone = &all_gone;
+ barrier();
+ if (atomic_read(&ioc_count))
+ complete(ioc_gone);
+ synchronize_rcu();
cfq_slab_kill();
}
git.openpandora.org / pandora-kernel.git / blobdiff