block/blk-cgroup.c

   1 /*
   2  * Common Block IO controller cgroup interface
   3  *
   4  * Based on ideas and code from CFQ, CFS and BFQ:
   5  * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
   6  *
   7  * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
   8  *                    Paolo Valente <paolo.valente@unimore.it>
   9  *
  10  * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  11  *                    Nauman Rafique <nauman@google.com>
  12  */
  13 #include <linux/ioprio.h>
  14 #include <linux/seq_file.h>
  15 #include <linux/kdev_t.h>
  16 #include <linux/module.h>
  17 #include <linux/err.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/slab.h>
  20 #include "blk-cgroup.h"
  21 #include <linux/genhd.h>
  22
  23 #define MAX_KEY_LEN 100
  24
  25 static DEFINE_SPINLOCK(blkio_list_lock);
  26 static LIST_HEAD(blkio_list);
  27
  28 struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
  29 EXPORT_SYMBOL_GPL(blkio_root_cgroup);
  30
  31 static struct cgroup_subsys_state *blkiocg_create(struct cgroup_subsys *,
  32                                                   struct cgroup *);
  33 static int blkiocg_can_attach(struct cgroup_subsys *, struct cgroup *,
  34                               struct task_struct *, bool);
  35 static void blkiocg_attach(struct cgroup_subsys *, struct cgroup *,
  36                            struct cgroup *, struct task_struct *, bool);
  37 static void blkiocg_destroy(struct cgroup_subsys *, struct cgroup *);
  38 static int blkiocg_populate(struct cgroup_subsys *, struct cgroup *);
  39
  40 /* for encoding cft->private value on file */
  41 #define BLKIOFILE_PRIVATE(x, val)       (((x) << 16) | (val))
  42 /* What policy owns the file, proportional or throttle */
  43 #define BLKIOFILE_POLICY(val)           (((val) >> 16) & 0xffff)
  44 #define BLKIOFILE_ATTR(val)             ((val) & 0xffff)
  45
  46 struct cgroup_subsys blkio_subsys = {
  47         .name = "blkio",
  48         .create = blkiocg_create,
  49         .can_attach = blkiocg_can_attach,
  50         .attach = blkiocg_attach,
  51         .destroy = blkiocg_destroy,
  52         .populate = blkiocg_populate,
  53 #ifdef CONFIG_BLK_CGROUP
  54         /* note: blkio_subsys_id is otherwise defined in blk-cgroup.h */
  55         .subsys_id = blkio_subsys_id,
  56 #endif
  57         .use_id = 1,
  58         .module = THIS_MODULE,
  59 };
  60 EXPORT_SYMBOL_GPL(blkio_subsys);
  61
  62 static inline void blkio_policy_insert_node(struct blkio_cgroup *blkcg,
  63                                             struct blkio_policy_node *pn)
  64 {
  65         list_add(&pn->node, &blkcg->policy_list);
  66 }
  67
  68 static inline bool cftype_blkg_same_policy(struct cftype *cft,
  69                         struct blkio_group *blkg)
  70 {
  71         enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  72
  73         if (blkg->plid == plid)
  74                 return 1;
  75
  76         return 0;
  77 }
  78
  79 /* Determines if policy node matches cgroup file being accessed */
  80 static inline bool pn_matches_cftype(struct cftype *cft,
  81                         struct blkio_policy_node *pn)
  82 {
  83         enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  84         int fileid = BLKIOFILE_ATTR(cft->private);
  85
  86         return (plid == pn->plid && fileid == pn->fileid);
  87 }
  88
  89 /* Must be called with blkcg->lock held */
  90 static inline void blkio_policy_delete_node(struct blkio_policy_node *pn)
  91 {
  92         list_del(&pn->node);
  93 }
  94
  95 /* Must be called with blkcg->lock held */
  96 static struct blkio_policy_node *
  97 blkio_policy_search_node(const struct blkio_cgroup *blkcg, dev_t dev,
  98                 enum blkio_policy_id plid, int fileid)
  99 {
 100         struct blkio_policy_node *pn;
 101
 102         list_for_each_entry(pn, &blkcg->policy_list, node) {
 103                 if (pn->dev == dev && pn->plid == plid && pn->fileid == fileid)
 104                         return pn;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
 111 {
 112         return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
 113                             struct blkio_cgroup, css);
 114 }
 115 EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
 116
 117 static inline void
 118 blkio_update_group_weight(struct blkio_group *blkg, unsigned int weight)
 119 {
 120         struct blkio_policy_type *blkiop;
 121
 122         list_for_each_entry(blkiop, &blkio_list, list) {
 123                 /* If this policy does not own the blkg, do not send updates */
 124                 if (blkiop->plid != blkg->plid)
 125                         continue;
 126                 if (blkiop->ops.blkio_update_group_weight_fn)
 127                         blkiop->ops.blkio_update_group_weight_fn(blkg->key,
 128                                                         blkg, weight);
 129         }
 130 }
 131
 132 static inline void blkio_update_group_bps(struct blkio_group *blkg, u64 bps,
 133                                 int fileid)
 134 {
 135         struct blkio_policy_type *blkiop;
 136
 137         list_for_each_entry(blkiop, &blkio_list, list) {
 138
 139                 /* If this policy does not own the blkg, do not send updates */
 140                 if (blkiop->plid != blkg->plid)
 141                         continue;
 142
 143                 if (fileid == BLKIO_THROTL_read_bps_device
 144                     && blkiop->ops.blkio_update_group_read_bps_fn)
 145                         blkiop->ops.blkio_update_group_read_bps_fn(blkg->key,
 146                                                                 blkg, bps);
 147
 148                 if (fileid == BLKIO_THROTL_write_bps_device
 149                     && blkiop->ops.blkio_update_group_write_bps_fn)
 150                         blkiop->ops.blkio_update_group_write_bps_fn(blkg->key,
 151                                                                 blkg, bps);
 152         }
 153 }
 154
 155 static inline void blkio_update_group_iops(struct blkio_group *blkg,
 156                         unsigned int iops, int fileid)
 157 {
 158         struct blkio_policy_type *blkiop;
 159
 160         list_for_each_entry(blkiop, &blkio_list, list) {
 161
 162                 /* If this policy does not own the blkg, do not send updates */
 163                 if (blkiop->plid != blkg->plid)
 164                         continue;
 165
 166                 if (fileid == BLKIO_THROTL_read_iops_device
 167                     && blkiop->ops.blkio_update_group_read_iops_fn)
 168                         blkiop->ops.blkio_update_group_read_iops_fn(blkg->key,
 169                                                                 blkg, iops);
 170
 171                 if (fileid == BLKIO_THROTL_write_iops_device
 172                     && blkiop->ops.blkio_update_group_write_iops_fn)
 173                         blkiop->ops.blkio_update_group_write_iops_fn(blkg->key,
 174                                                                 blkg,iops);
 175         }
 176 }
 177
 178 /*
 179  * Add to the appropriate stat variable depending on the request type.
 180  * This should be called with the blkg->stats_lock held.
 181  */
 182 static void blkio_add_stat(uint64_t *stat, uint64_t add, bool direction,
 183                                 bool sync)
 184 {
 185         if (direction)
 186                 stat[BLKIO_STAT_WRITE] += add;
 187         else
 188                 stat[BLKIO_STAT_READ] += add;
 189         if (sync)
 190                 stat[BLKIO_STAT_SYNC] += add;
 191         else
 192                 stat[BLKIO_STAT_ASYNC] += add;
 193 }
 194
 195 /*
 196  * Decrements the appropriate stat variable if non-zero depending on the
 197  * request type. Panics on value being zero.
 198  * This should be called with the blkg->stats_lock held.
 199  */
 200 static void blkio_check_and_dec_stat(uint64_t *stat, bool direction, bool sync)
 201 {
 202         if (direction) {
 203                 BUG_ON(stat[BLKIO_STAT_WRITE] == 0);
 204                 stat[BLKIO_STAT_WRITE]--;
 205         } else {
 206                 BUG_ON(stat[BLKIO_STAT_READ] == 0);
 207                 stat[BLKIO_STAT_READ]--;
 208         }
 209         if (sync) {
 210                 BUG_ON(stat[BLKIO_STAT_SYNC] == 0);
 211                 stat[BLKIO_STAT_SYNC]--;
 212         } else {
 213                 BUG_ON(stat[BLKIO_STAT_ASYNC] == 0);
 214                 stat[BLKIO_STAT_ASYNC]--;
 215         }
 216 }
 217
 218 #ifdef CONFIG_DEBUG_BLK_CGROUP
 219 /* This should be called with the blkg->stats_lock held. */
 220 static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
 221                                                 struct blkio_group *curr_blkg)
 222 {
 223         if (blkio_blkg_waiting(&blkg->stats))
 224                 return;
 225         if (blkg == curr_blkg)
 226                 return;
 227         blkg->stats.start_group_wait_time = sched_clock();
 228         blkio_mark_blkg_waiting(&blkg->stats);
 229 }
 230
 231 /* This should be called with the blkg->stats_lock held. */
 232 static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
 233 {
 234         unsigned long long now;
 235
 236         if (!blkio_blkg_waiting(stats))
 237                 return;
 238
 239         now = sched_clock();
 240         if (time_after64(now, stats->start_group_wait_time))
 241                 stats->group_wait_time += now - stats->start_group_wait_time;
 242         blkio_clear_blkg_waiting(stats);
 243 }
 244
 245 /* This should be called with the blkg->stats_lock held. */
 246 static void blkio_end_empty_time(struct blkio_group_stats *stats)
 247 {
 248         unsigned long long now;
 249
 250         if (!blkio_blkg_empty(stats))
 251                 return;
 252
 253         now = sched_clock();
 254         if (time_after64(now, stats->start_empty_time))
 255                 stats->empty_time += now - stats->start_empty_time;
 256         blkio_clear_blkg_empty(stats);
 257 }
 258
 259 void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg)
 260 {
 261         unsigned long flags;
 262
 263         spin_lock_irqsave(&blkg->stats_lock, flags);
 264         BUG_ON(blkio_blkg_idling(&blkg->stats));
 265         blkg->stats.start_idle_time = sched_clock();
 266         blkio_mark_blkg_idling(&blkg->stats);
 267         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 268 }
 269 EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);
 270
 271 void blkiocg_update_idle_time_stats(struct blkio_group *blkg)
 272 {
 273         unsigned long flags;
 274         unsigned long long now;
 275         struct blkio_group_stats *stats;
 276
 277         spin_lock_irqsave(&blkg->stats_lock, flags);
 278         stats = &blkg->stats;
 279         if (blkio_blkg_idling(stats)) {
 280                 now = sched_clock();
 281                 if (time_after64(now, stats->start_idle_time))
 282                         stats->idle_time += now - stats->start_idle_time;
 283                 blkio_clear_blkg_idling(stats);
 284         }
 285         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 286 }
 287 EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);
 288
 289 void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg)
 290 {
 291         unsigned long flags;
 292         struct blkio_group_stats *stats;
 293
 294         spin_lock_irqsave(&blkg->stats_lock, flags);
 295         stats = &blkg->stats;
 296         stats->avg_queue_size_sum +=
 297                         stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] +
 298                         stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE];
 299         stats->avg_queue_size_samples++;
 300         blkio_update_group_wait_time(stats);
 301         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 302 }
 303 EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);
 304
 305 void blkiocg_set_start_empty_time(struct blkio_group *blkg)
 306 {
 307         unsigned long flags;
 308         struct blkio_group_stats *stats;
 309
 310         spin_lock_irqsave(&blkg->stats_lock, flags);
 311         stats = &blkg->stats;
 312
 313         if (stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] ||
 314                         stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE]) {
 315                 spin_unlock_irqrestore(&blkg->stats_lock, flags);
 316                 return;
 317         }
 318
 319         /*
 320          * group is already marked empty. This can happen if cfqq got new
 321          * request in parent group and moved to this group while being added
 322          * to service tree. Just ignore the event and move on.
 323          */
 324         if(blkio_blkg_empty(stats)) {
 325                 spin_unlock_irqrestore(&blkg->stats_lock, flags);
 326                 return;
 327         }
 328
 329         stats->start_empty_time = sched_clock();
 330         blkio_mark_blkg_empty(stats);
 331         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 332 }
 333 EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);
 334
 335 void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
 336                         unsigned long dequeue)
 337 {
 338         blkg->stats.dequeue += dequeue;
 339 }
 340 EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
 341 #else
 342 static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
 343                                         struct blkio_group *curr_blkg) {}
 344 static inline void blkio_end_empty_time(struct blkio_group_stats *stats) {}
 345 #endif
 346
 347 void blkiocg_update_io_add_stats(struct blkio_group *blkg,
 348                         struct blkio_group *curr_blkg, bool direction,
 349                         bool sync)
 350 {
 351         unsigned long flags;
 352
 353         spin_lock_irqsave(&blkg->stats_lock, flags);
 354         blkio_add_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED], 1, direction,
 355                         sync);
 356         blkio_end_empty_time(&blkg->stats);
 357         blkio_set_start_group_wait_time(blkg, curr_blkg);
 358         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 359 }
 360 EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);
 361
 362 void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
 363                                                 bool direction, bool sync)
 364 {
 365         unsigned long flags;
 366
 367         spin_lock_irqsave(&blkg->stats_lock, flags);
 368         blkio_check_and_dec_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED],
 369                                         direction, sync);
 370         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 371 }
 372 EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);
 373
 374 void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time,
 375                                 unsigned long unaccounted_time)
 376 {
 377         unsigned long flags;
 378
 379         spin_lock_irqsave(&blkg->stats_lock, flags);
 380         blkg->stats.time += time;
 381         blkg->stats.unaccounted_time += unaccounted_time;
 382         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 383 }
 384 EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);
 385
 386 void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
 387                                 uint64_t bytes, bool direction, bool sync)
 388 {
 389         struct blkio_group_stats *stats;
 390         unsigned long flags;
 391
 392         spin_lock_irqsave(&blkg->stats_lock, flags);
 393         stats = &blkg->stats;
 394         stats->sectors += bytes >> 9;
 395         blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICED], 1, direction,
 396                         sync);
 397         blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICE_BYTES], bytes,
 398                         direction, sync);
 399         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 400 }
 401 EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
 402
 403 void blkiocg_update_completion_stats(struct blkio_group *blkg,
 404         uint64_t start_time, uint64_t io_start_time, bool direction, bool sync)
 405 {
 406         struct blkio_group_stats *stats;
 407         unsigned long flags;
 408         unsigned long long now = sched_clock();
 409
 410         spin_lock_irqsave(&blkg->stats_lock, flags);
 411         stats = &blkg->stats;
 412         if (time_after64(now, io_start_time))
 413                 blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICE_TIME],
 414                                 now - io_start_time, direction, sync);
 415         if (time_after64(io_start_time, start_time))
 416                 blkio_add_stat(stats->stat_arr[BLKIO_STAT_WAIT_TIME],
 417                                 io_start_time - start_time, direction, sync);
 418         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 419 }
 420 EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
 421
 422 void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction,
 423                                         bool sync)
 424 {
 425         unsigned long flags;
 426
 427         spin_lock_irqsave(&blkg->stats_lock, flags);
 428         blkio_add_stat(blkg->stats.stat_arr[BLKIO_STAT_MERGED], 1, direction,
 429                         sync);
 430         spin_unlock_irqrestore(&blkg->stats_lock, flags);
 431 }
 432 EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);
 433
 434 void blkiocg_add_blkio_group(struct blkio_cgroup *blkcg,
 435                 struct blkio_group *blkg, void *key, dev_t dev,
 436                 enum blkio_policy_id plid)
 437 {
 438         unsigned long flags;
 439
 440         spin_lock_irqsave(&blkcg->lock, flags);
 441         spin_lock_init(&blkg->stats_lock);
 442         rcu_assign_pointer(blkg->key, key);
 443         blkg->blkcg_id = css_id(&blkcg->css);
 444         hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
 445         blkg->plid = plid;
 446         spin_unlock_irqrestore(&blkcg->lock, flags);
 447         /* Need to take css reference ? */
 448         cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
 449         blkg->dev = dev;
 450 }
 451 EXPORT_SYMBOL_GPL(blkiocg_add_blkio_group);
 452
 453 static void __blkiocg_del_blkio_group(struct blkio_group *blkg)
 454 {
 455         hlist_del_init_rcu(&blkg->blkcg_node);
 456         blkg->blkcg_id = 0;
 457 }
 458
 459 /*
 460  * returns 0 if blkio_group was still on cgroup list. Otherwise returns 1
 461  * indicating that blk_group was unhashed by the time we got to it.
 462  */
 463 int blkiocg_del_blkio_group(struct blkio_group *blkg)
 464 {
 465         struct blkio_cgroup *blkcg;
 466         unsigned long flags;
 467         struct cgroup_subsys_state *css;
 468         int ret = 1;
 469
 470         rcu_read_lock();
 471         css = css_lookup(&blkio_subsys, blkg->blkcg_id);
 472         if (css) {
 473                 blkcg = container_of(css, struct blkio_cgroup, css);
 474                 spin_lock_irqsave(&blkcg->lock, flags);
 475                 if (!hlist_unhashed(&blkg->blkcg_node)) {
 476                         __blkiocg_del_blkio_group(blkg);
 477                         ret = 0;
 478                 }
 479                 spin_unlock_irqrestore(&blkcg->lock, flags);
 480         }
 481
 482         rcu_read_unlock();
 483         return ret;
 484 }
 485 EXPORT_SYMBOL_GPL(blkiocg_del_blkio_group);
 486
 487 /* called under rcu_read_lock(). */
 488 struct blkio_group *blkiocg_lookup_group(struct blkio_cgroup *blkcg, void *key)
 489 {
 490         struct blkio_group *blkg;
 491         struct hlist_node *n;
 492         void *__key;
 493
 494         hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) {
 495                 __key = blkg->key;
 496                 if (__key == key)
 497                         return blkg;
 498         }
 499
 500         return NULL;
 501 }
 502 EXPORT_SYMBOL_GPL(blkiocg_lookup_group);
 503
 504 static int
 505 blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
 506 {
 507         struct blkio_cgroup *blkcg;
 508         struct blkio_group *blkg;
 509         struct blkio_group_stats *stats;
 510         struct hlist_node *n;
 511         uint64_t queued[BLKIO_STAT_TOTAL];
 512         int i;
 513 #ifdef CONFIG_DEBUG_BLK_CGROUP
 514         bool idling, waiting, empty;
 515         unsigned long long now = sched_clock();
 516 #endif
 517
 518         blkcg = cgroup_to_blkio_cgroup(cgroup);
 519         spin_lock_irq(&blkcg->lock);
 520         hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
 521                 spin_lock(&blkg->stats_lock);
 522                 stats = &blkg->stats;
 523 #ifdef CONFIG_DEBUG_BLK_CGROUP
 524                 idling = blkio_blkg_idling(stats);
 525                 waiting = blkio_blkg_waiting(stats);
 526                 empty = blkio_blkg_empty(stats);
 527 #endif
 528                 for (i = 0; i < BLKIO_STAT_TOTAL; i++)
 529                         queued[i] = stats->stat_arr[BLKIO_STAT_QUEUED][i];
 530                 memset(stats, 0, sizeof(struct blkio_group_stats));
 531                 for (i = 0; i < BLKIO_STAT_TOTAL; i++)
 532                         stats->stat_arr[BLKIO_STAT_QUEUED][i] = queued[i];
 533 #ifdef CONFIG_DEBUG_BLK_CGROUP
 534                 if (idling) {
 535                         blkio_mark_blkg_idling(stats);
 536                         stats->start_idle_time = now;
 537                 }
 538                 if (waiting) {
 539                         blkio_mark_blkg_waiting(stats);
 540                         stats->start_group_wait_time = now;
 541                 }
 542                 if (empty) {
 543                         blkio_mark_blkg_empty(stats);
 544                         stats->start_empty_time = now;
 545                 }
 546 #endif
 547                 spin_unlock(&blkg->stats_lock);
 548         }
 549         spin_unlock_irq(&blkcg->lock);
 550         return 0;
 551 }
 552
 553 static void blkio_get_key_name(enum stat_sub_type type, dev_t dev, char *str,
 554                                 int chars_left, bool diskname_only)
 555 {
 556         snprintf(str, chars_left, "%d:%d", MAJOR(dev), MINOR(dev));
 557         chars_left -= strlen(str);
 558         if (chars_left <= 0) {
 559                 printk(KERN_WARNING
 560                         "Possibly incorrect cgroup stat display format");
 561                 return;
 562         }
 563         if (diskname_only)
 564                 return;
 565         switch (type) {
 566         case BLKIO_STAT_READ:
 567                 strlcat(str, " Read", chars_left);
 568                 break;
 569         case BLKIO_STAT_WRITE:
 570                 strlcat(str, " Write", chars_left);
 571                 break;
 572         case BLKIO_STAT_SYNC:
 573                 strlcat(str, " Sync", chars_left);
 574                 break;
 575         case BLKIO_STAT_ASYNC:
 576                 strlcat(str, " Async", chars_left);
 577                 break;
 578         case BLKIO_STAT_TOTAL:
 579                 strlcat(str, " Total", chars_left);
 580                 break;
 581         default:
 582                 strlcat(str, " Invalid", chars_left);
 583         }
 584 }
 585
 586 static uint64_t blkio_fill_stat(char *str, int chars_left, uint64_t val,
 587                                 struct cgroup_map_cb *cb, dev_t dev)
 588 {
 589         blkio_get_key_name(0, dev, str, chars_left, true);
 590         cb->fill(cb, str, val);
 591         return val;
 592 }
 593
 594 /* This should be called with blkg->stats_lock held */
 595 static uint64_t blkio_get_stat(struct blkio_group *blkg,
 596                 struct cgroup_map_cb *cb, dev_t dev, enum stat_type type)
 597 {
 598         uint64_t disk_total;
 599         char key_str[MAX_KEY_LEN];
 600         enum stat_sub_type sub_type;
 601
 602         if (type == BLKIO_STAT_TIME)
 603                 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
 604                                         blkg->stats.time, cb, dev);
 605         if (type == BLKIO_STAT_SECTORS)
 606                 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
 607                                         blkg->stats.sectors, cb, dev);
 608 #ifdef CONFIG_DEBUG_BLK_CGROUP
 609         if (type == BLKIO_STAT_UNACCOUNTED_TIME)
 610                 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
 611                                         blkg->stats.unaccounted_time, cb, dev);
 612         if (type == BLKIO_STAT_AVG_QUEUE_SIZE) {
 613                 uint64_t sum = blkg->stats.avg_queue_size_sum;
 614                 uint64_t samples = blkg->stats.avg_queue_size_samples;
 615                 if (samples)
 616                         do_div(sum, samples);
 617                 else
 618                         sum = 0;
 619                 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, sum, cb, dev);
 620         }
 621         if (type == BLKIO_STAT_GROUP_WAIT_TIME)
 622                 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
 623                                         blkg->stats.group_wait_time, cb, dev);
 624         if (type == BLKIO_STAT_IDLE_TIME)
 625                 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
 626                                         blkg->stats.idle_time, cb, dev);
 627         if (type == BLKIO_STAT_EMPTY_TIME)
 628                 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
 629                                         blkg->stats.empty_time, cb, dev);
 630         if (type == BLKIO_STAT_DEQUEUE)
 631                 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
 632                                         blkg->stats.dequeue, cb, dev);
 633 #endif
 634
 635         for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
 636                         sub_type++) {
 637                 blkio_get_key_name(sub_type, dev, key_str, MAX_KEY_LEN, false);
 638                 cb->fill(cb, key_str, blkg->stats.stat_arr[type][sub_type]);
 639         }
 640         disk_total = blkg->stats.stat_arr[type][BLKIO_STAT_READ] +
 641                         blkg->stats.stat_arr[type][BLKIO_STAT_WRITE];
 642         blkio_get_key_name(BLKIO_STAT_TOTAL, dev, key_str, MAX_KEY_LEN, false);
 643         cb->fill(cb, key_str, disk_total);
 644         return disk_total;
 645 }
 646
 647 static int blkio_check_dev_num(dev_t dev)
 648 {
 649         int part = 0;
 650         struct gendisk *disk;
 651
 652         disk = get_gendisk(dev, &part);
 653         if (!disk || part)
 654                 return -ENODEV;
 655
 656         return 0;
 657 }
 658
 659 static int blkio_policy_parse_and_set(char *buf,
 660         struct blkio_policy_node *newpn, enum blkio_policy_id plid, int fileid)
 661 {
 662         char *s[4], *p, *major_s = NULL, *minor_s = NULL;
 663         int ret;
 664         unsigned long major, minor, temp;
 665         int i = 0;
 666         dev_t dev;
 667         u64 bps, iops;
 668
 669         memset(s, 0, sizeof(s));
 670
 671         while ((p = strsep(&buf, " ")) != NULL) {
 672                 if (!*p)
 673                         continue;
 674
 675                 s[i++] = p;
 676
 677                 /* Prevent from inputing too many things */
 678                 if (i == 3)
 679                         break;
 680         }
 681
 682         if (i != 2)
 683                 return -EINVAL;
 684
 685         p = strsep(&s[0], ":");
 686         if (p != NULL)
 687                 major_s = p;
 688         else
 689                 return -EINVAL;
 690
 691         minor_s = s[0];
 692         if (!minor_s)
 693                 return -EINVAL;
 694
 695         ret = strict_strtoul(major_s, 10, &major);
 696         if (ret)
 697                 return -EINVAL;
 698
 699         ret = strict_strtoul(minor_s, 10, &minor);
 700         if (ret)
 701                 return -EINVAL;
 702
 703         dev = MKDEV(major, minor);
 704
 705         ret = blkio_check_dev_num(dev);
 706         if (ret)
 707                 return ret;
 708
 709         newpn->dev = dev;
 710
 711         if (s[1] == NULL)
 712                 return -EINVAL;
 713
 714         switch (plid) {
 715         case BLKIO_POLICY_PROP:
 716                 ret = strict_strtoul(s[1], 10, &temp);
 717                 if (ret || (temp < BLKIO_WEIGHT_MIN && temp > 0) ||
 718                         temp > BLKIO_WEIGHT_MAX)
 719                         return -EINVAL;
 720
 721                 newpn->plid = plid;
 722                 newpn->fileid = fileid;
 723                 newpn->val.weight = temp;
 724                 break;
 725         case BLKIO_POLICY_THROTL:
 726                 switch(fileid) {
 727                 case BLKIO_THROTL_read_bps_device:
 728                 case BLKIO_THROTL_write_bps_device:
 729                         ret = strict_strtoull(s[1], 10, &bps);
 730                         if (ret)
 731                                 return -EINVAL;
 732
 733                         newpn->plid = plid;
 734                         newpn->fileid = fileid;
 735                         newpn->val.bps = bps;
 736                         break;
 737                 case BLKIO_THROTL_read_iops_device:
 738                 case BLKIO_THROTL_write_iops_device:
 739                         ret = strict_strtoull(s[1], 10, &iops);
 740                         if (ret)
 741                                 return -EINVAL;
 742
 743                         if (iops > THROTL_IOPS_MAX)
 744                                 return -EINVAL;
 745
 746                         newpn->plid = plid;
 747                         newpn->fileid = fileid;
 748                         newpn->val.iops = (unsigned int)iops;
 749                         break;
 750                 }
 751                 break;
 752         default:
 753                 BUG();
 754         }
 755
 756         return 0;
 757 }
 758
 759 unsigned int blkcg_get_weight(struct blkio_cgroup *blkcg,
 760                               dev_t dev)
 761 {
 762         struct blkio_policy_node *pn;
 763
 764         pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_PROP,
 765                                 BLKIO_PROP_weight_device);
 766         if (pn)
 767                 return pn->val.weight;
 768         else
 769                 return blkcg->weight;
 770 }
 771 EXPORT_SYMBOL_GPL(blkcg_get_weight);
 772
 773 uint64_t blkcg_get_read_bps(struct blkio_cgroup *blkcg, dev_t dev)
 774 {
 775         struct blkio_policy_node *pn;
 776
 777         pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
 778                                 BLKIO_THROTL_read_bps_device);
 779         if (pn)
 780                 return pn->val.bps;
 781         else
 782                 return -1;
 783 }
 784
 785 uint64_t blkcg_get_write_bps(struct blkio_cgroup *blkcg, dev_t dev)
 786 {
 787         struct blkio_policy_node *pn;
 788         pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
 789                                 BLKIO_THROTL_write_bps_device);
 790         if (pn)
 791                 return pn->val.bps;
 792         else
 793                 return -1;
 794 }
 795
 796 unsigned int blkcg_get_read_iops(struct blkio_cgroup *blkcg, dev_t dev)
 797 {
 798         struct blkio_policy_node *pn;
 799
 800         pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
 801                                 BLKIO_THROTL_read_iops_device);
 802         if (pn)
 803                 return pn->val.iops;
 804         else
 805                 return -1;
 806 }
 807
 808 unsigned int blkcg_get_write_iops(struct blkio_cgroup *blkcg, dev_t dev)
 809 {
 810         struct blkio_policy_node *pn;
 811         pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
 812                                 BLKIO_THROTL_write_iops_device);
 813         if (pn)
 814                 return pn->val.iops;
 815         else
 816                 return -1;
 817 }
 818
 819 /* Checks whether user asked for deleting a policy rule */
 820 static bool blkio_delete_rule_command(struct blkio_policy_node *pn)
 821 {
 822         switch(pn->plid) {
 823         case BLKIO_POLICY_PROP:
 824                 if (pn->val.weight == 0)
 825                         return 1;
 826                 break;
 827         case BLKIO_POLICY_THROTL:
 828                 switch(pn->fileid) {
 829                 case BLKIO_THROTL_read_bps_device:
 830                 case BLKIO_THROTL_write_bps_device:
 831                         if (pn->val.bps == 0)
 832                                 return 1;
 833                         break;
 834                 case BLKIO_THROTL_read_iops_device:
 835                 case BLKIO_THROTL_write_iops_device:
 836                         if (pn->val.iops == 0)
 837                                 return 1;
 838                 }
 839                 break;
 840         default:
 841                 BUG();
 842         }
 843
 844         return 0;
 845 }
 846
 847 static void blkio_update_policy_rule(struct blkio_policy_node *oldpn,
 848                                         struct blkio_policy_node *newpn)
 849 {
 850         switch(oldpn->plid) {
 851         case BLKIO_POLICY_PROP:
 852                 oldpn->val.weight = newpn->val.weight;
 853                 break;
 854         case BLKIO_POLICY_THROTL:
 855                 switch(newpn->fileid) {
 856                 case BLKIO_THROTL_read_bps_device:
 857                 case BLKIO_THROTL_write_bps_device:
 858                         oldpn->val.bps = newpn->val.bps;
 859                         break;
 860                 case BLKIO_THROTL_read_iops_device:
 861                 case BLKIO_THROTL_write_iops_device:
 862                         oldpn->val.iops = newpn->val.iops;
 863                 }
 864                 break;
 865         default:
 866                 BUG();
 867         }
 868 }
 869
 870 /*
 871  * Some rules/values in blkg have changed. Propagate those to respective
 872  * policies.
 873  */
 874 static void blkio_update_blkg_policy(struct blkio_cgroup *blkcg,
 875                 struct blkio_group *blkg, struct blkio_policy_node *pn)
 876 {
 877         unsigned int weight, iops;
 878         u64 bps;
 879
 880         switch(pn->plid) {
 881         case BLKIO_POLICY_PROP:
 882                 weight = pn->val.weight ? pn->val.weight :
 883                                 blkcg->weight;
 884                 blkio_update_group_weight(blkg, weight);
 885                 break;
 886         case BLKIO_POLICY_THROTL:
 887                 switch(pn->fileid) {
 888                 case BLKIO_THROTL_read_bps_device:
 889                 case BLKIO_THROTL_write_bps_device:
 890                         bps = pn->val.bps ? pn->val.bps : (-1);
 891                         blkio_update_group_bps(blkg, bps, pn->fileid);
 892                         break;
 893                 case BLKIO_THROTL_read_iops_device:
 894                 case BLKIO_THROTL_write_iops_device:
 895                         iops = pn->val.iops ? pn->val.iops : (-1);
 896                         blkio_update_group_iops(blkg, iops, pn->fileid);
 897                         break;
 898                 }
 899                 break;
 900         default:
 901                 BUG();
 902         }
 903 }
 904
 905 /*
 906  * A policy node rule has been updated. Propagate this update to all the
 907  * block groups which might be affected by this update.
 908  */
 909 static void blkio_update_policy_node_blkg(struct blkio_cgroup *blkcg,
 910                                 struct blkio_policy_node *pn)
 911 {
 912         struct blkio_group *blkg;
 913         struct hlist_node *n;
 914
 915         spin_lock(&blkio_list_lock);
 916         spin_lock_irq(&blkcg->lock);
 917
 918         hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
 919                 if (pn->dev != blkg->dev || pn->plid != blkg->plid)
 920                         continue;
 921                 blkio_update_blkg_policy(blkcg, blkg, pn);
 922         }
 923
 924         spin_unlock_irq(&blkcg->lock);
 925         spin_unlock(&blkio_list_lock);
 926 }
 927
 928 static int blkiocg_file_write(struct cgroup *cgrp, struct cftype *cft,
 929                                        const char *buffer)
 930 {
 931         int ret = 0;
 932         char *buf;
 933         struct blkio_policy_node *newpn, *pn;
 934         struct blkio_cgroup *blkcg;
 935         int keep_newpn = 0;
 936         enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
 937         int fileid = BLKIOFILE_ATTR(cft->private);
 938
 939         buf = kstrdup(buffer, GFP_KERNEL);
 940         if (!buf)
 941                 return -ENOMEM;
 942
 943         newpn = kzalloc(sizeof(*newpn), GFP_KERNEL);
 944         if (!newpn) {
 945                 ret = -ENOMEM;
 946                 goto free_buf;
 947         }
 948
 949         ret = blkio_policy_parse_and_set(buf, newpn, plid, fileid);
 950         if (ret)
 951                 goto free_newpn;
 952
 953         blkcg = cgroup_to_blkio_cgroup(cgrp);
 954
 955         spin_lock_irq(&blkcg->lock);
 956
 957         pn = blkio_policy_search_node(blkcg, newpn->dev, plid, fileid);
 958         if (!pn) {
 959                 if (!blkio_delete_rule_command(newpn)) {
 960                         blkio_policy_insert_node(blkcg, newpn);
 961                         keep_newpn = 1;
 962                 }
 963                 spin_unlock_irq(&blkcg->lock);
 964                 goto update_io_group;
 965         }
 966
 967         if (blkio_delete_rule_command(newpn)) {
 968                 blkio_policy_delete_node(pn);
 969                 spin_unlock_irq(&blkcg->lock);
 970                 goto update_io_group;
 971         }
 972         spin_unlock_irq(&blkcg->lock);
 973
 974         blkio_update_policy_rule(pn, newpn);
 975
 976 update_io_group:
 977         blkio_update_policy_node_blkg(blkcg, newpn);
 978
 979 free_newpn:
 980         if (!keep_newpn)
 981                 kfree(newpn);
 982 free_buf:
 983         kfree(buf);
 984         return ret;
 985 }
 986
 987 static void
 988 blkio_print_policy_node(struct seq_file *m, struct blkio_policy_node *pn)
 989 {
 990         switch(pn->plid) {
 991                 case BLKIO_POLICY_PROP:
 992                         if (pn->fileid == BLKIO_PROP_weight_device)
 993                                 seq_printf(m, "%u:%u\t%u\n", MAJOR(pn->dev),
 994                                         MINOR(pn->dev), pn->val.weight);
 995                         break;
 996                 case BLKIO_POLICY_THROTL:
 997                         switch(pn->fileid) {
 998                         case BLKIO_THROTL_read_bps_device:
 999                         case BLKIO_THROTL_write_bps_device:
1000                                 seq_printf(m, "%u:%u\t%llu\n", MAJOR(pn->dev),
1001                                         MINOR(pn->dev), pn->val.bps);
1002                                 break;
1003                         case BLKIO_THROTL_read_iops_device:
1004                         case BLKIO_THROTL_write_iops_device:
1005                                 seq_printf(m, "%u:%u\t%u\n", MAJOR(pn->dev),
1006                                         MINOR(pn->dev), pn->val.iops);
1007                                 break;
1008                         }
1009                         break;
1010                 default:
1011                         BUG();
1012         }
1013 }
1014
1015 /* cgroup files which read their data from policy nodes end up here */
1016 static void blkio_read_policy_node_files(struct cftype *cft,
1017                         struct blkio_cgroup *blkcg, struct seq_file *m)
1018 {
1019         struct blkio_policy_node *pn;
1020
1021         if (!list_empty(&blkcg->policy_list)) {
1022                 spin_lock_irq(&blkcg->lock);
1023                 list_for_each_entry(pn, &blkcg->policy_list, node) {
1024                         if (!pn_matches_cftype(cft, pn))
1025                                 continue;
1026                         blkio_print_policy_node(m, pn);
1027                 }
1028                 spin_unlock_irq(&blkcg->lock);
1029         }
1030 }
1031
1032 static int blkiocg_file_read(struct cgroup *cgrp, struct cftype *cft,
1033                                 struct seq_file *m)
1034 {
1035         struct blkio_cgroup *blkcg;
1036         enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1037         int name = BLKIOFILE_ATTR(cft->private);
1038
1039         blkcg = cgroup_to_blkio_cgroup(cgrp);
1040
1041         switch(plid) {
1042         case BLKIO_POLICY_PROP:
1043                 switch(name) {
1044                 case BLKIO_PROP_weight_device:
1045                         blkio_read_policy_node_files(cft, blkcg, m);
1046                         return 0;
1047                 default:
1048                         BUG();
1049                 }
1050                 break;
1051         case BLKIO_POLICY_THROTL:
1052                 switch(name){
1053                 case BLKIO_THROTL_read_bps_device:
1054                 case BLKIO_THROTL_write_bps_device:
1055                 case BLKIO_THROTL_read_iops_device:
1056                 case BLKIO_THROTL_write_iops_device:
1057                         blkio_read_policy_node_files(cft, blkcg, m);
1058                         return 0;
1059                 default:
1060                         BUG();
1061                 }
1062                 break;
1063         default:
1064                 BUG();
1065         }
1066
1067         return 0;
1068 }
1069
1070 static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg,
1071                 struct cftype *cft, struct cgroup_map_cb *cb, enum stat_type type,
1072                 bool show_total)
1073 {
1074         struct blkio_group *blkg;
1075         struct hlist_node *n;
1076         uint64_t cgroup_total = 0;
1077
1078         rcu_read_lock();
1079         hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) {
1080                 if (blkg->dev) {
1081                         if (!cftype_blkg_same_policy(cft, blkg))
1082                                 continue;
1083                         spin_lock_irq(&blkg->stats_lock);
1084                         cgroup_total += blkio_get_stat(blkg, cb, blkg->dev,
1085                                                 type);
1086                         spin_unlock_irq(&blkg->stats_lock);
1087                 }
1088         }
1089         if (show_total)
1090                 cb->fill(cb, "Total", cgroup_total);
1091         rcu_read_unlock();
1092         return 0;
1093 }
1094
1095 /* All map kind of cgroup file get serviced by this function */
1096 static int blkiocg_file_read_map(struct cgroup *cgrp, struct cftype *cft,
1097                                 struct cgroup_map_cb *cb)
1098 {
1099         struct blkio_cgroup *blkcg;
1100         enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1101         int name = BLKIOFILE_ATTR(cft->private);
1102
1103         blkcg = cgroup_to_blkio_cgroup(cgrp);
1104
1105         switch(plid) {
1106         case BLKIO_POLICY_PROP:
1107                 switch(name) {
1108                 case BLKIO_PROP_time:
1109                         return blkio_read_blkg_stats(blkcg, cft, cb,
1110                                                 BLKIO_STAT_TIME, 0);
1111                 case BLKIO_PROP_sectors:
1112                         return blkio_read_blkg_stats(blkcg, cft, cb,
1113                                                 BLKIO_STAT_SECTORS, 0);
1114                 case BLKIO_PROP_io_service_bytes:
1115                         return blkio_read_blkg_stats(blkcg, cft, cb,
1116                                                 BLKIO_STAT_SERVICE_BYTES, 1);
1117                 case BLKIO_PROP_io_serviced:
1118                         return blkio_read_blkg_stats(blkcg, cft, cb,
1119                                                 BLKIO_STAT_SERVICED, 1);
1120                 case BLKIO_PROP_io_service_time:
1121                         return blkio_read_blkg_stats(blkcg, cft, cb,
1122                                                 BLKIO_STAT_SERVICE_TIME, 1);
1123                 case BLKIO_PROP_io_wait_time:
1124                         return blkio_read_blkg_stats(blkcg, cft, cb,
1125                                                 BLKIO_STAT_WAIT_TIME, 1);
1126                 case BLKIO_PROP_io_merged:
1127                         return blkio_read_blkg_stats(blkcg, cft, cb,
1128                                                 BLKIO_STAT_MERGED, 1);
1129                 case BLKIO_PROP_io_queued:
1130                         return blkio_read_blkg_stats(blkcg, cft, cb,
1131                                                 BLKIO_STAT_QUEUED, 1);
1132 #ifdef CONFIG_DEBUG_BLK_CGROUP
1133                 case BLKIO_PROP_unaccounted_time:
1134                         return blkio_read_blkg_stats(blkcg, cft, cb,
1135                                                 BLKIO_STAT_UNACCOUNTED_TIME, 0);
1136                 case BLKIO_PROP_dequeue:
1137                         return blkio_read_blkg_stats(blkcg, cft, cb,
1138                                                 BLKIO_STAT_DEQUEUE, 0);
1139                 case BLKIO_PROP_avg_queue_size:
1140                         return blkio_read_blkg_stats(blkcg, cft, cb,
1141                                                 BLKIO_STAT_AVG_QUEUE_SIZE, 0);
1142                 case BLKIO_PROP_group_wait_time:
1143                         return blkio_read_blkg_stats(blkcg, cft, cb,
1144                                                 BLKIO_STAT_GROUP_WAIT_TIME, 0);
1145                 case BLKIO_PROP_idle_time:
1146                         return blkio_read_blkg_stats(blkcg, cft, cb,
1147                                                 BLKIO_STAT_IDLE_TIME, 0);
1148                 case BLKIO_PROP_empty_time:
1149                         return blkio_read_blkg_stats(blkcg, cft, cb,
1150                                                 BLKIO_STAT_EMPTY_TIME, 0);
1151 #endif
1152                 default:
1153                         BUG();
1154                 }
1155                 break;
1156         case BLKIO_POLICY_THROTL:
1157                 switch(name){
1158                 case BLKIO_THROTL_io_service_bytes:
1159                         return blkio_read_blkg_stats(blkcg, cft, cb,
1160                                                 BLKIO_STAT_SERVICE_BYTES, 1);
1161                 case BLKIO_THROTL_io_serviced:
1162                         return blkio_read_blkg_stats(blkcg, cft, cb,
1163                                                 BLKIO_STAT_SERVICED, 1);
1164                 default:
1165                         BUG();
1166                 }
1167                 break;
1168         default:
1169                 BUG();
1170         }
1171
1172         return 0;
1173 }
1174
1175 static int blkio_weight_write(struct blkio_cgroup *blkcg, u64 val)
1176 {
1177         struct blkio_group *blkg;
1178         struct hlist_node *n;
1179         struct blkio_policy_node *pn;
1180
1181         if (val < BLKIO_WEIGHT_MIN || val > BLKIO_WEIGHT_MAX)
1182                 return -EINVAL;
1183
1184         spin_lock(&blkio_list_lock);
1185         spin_lock_irq(&blkcg->lock);
1186         blkcg->weight = (unsigned int)val;
1187
1188         hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
1189                 pn = blkio_policy_search_node(blkcg, blkg->dev,
1190                                 BLKIO_POLICY_PROP, BLKIO_PROP_weight_device);
1191                 if (pn)
1192                         continue;
1193
1194                 blkio_update_group_weight(blkg, blkcg->weight);
1195         }
1196         spin_unlock_irq(&blkcg->lock);
1197         spin_unlock(&blkio_list_lock);
1198         return 0;
1199 }
1200
1201 static u64 blkiocg_file_read_u64 (struct cgroup *cgrp, struct cftype *cft) {
1202         struct blkio_cgroup *blkcg;
1203         enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1204         int name = BLKIOFILE_ATTR(cft->private);
1205
1206         blkcg = cgroup_to_blkio_cgroup(cgrp);
1207
1208         switch(plid) {
1209         case BLKIO_POLICY_PROP:
1210                 switch(name) {
1211                 case BLKIO_PROP_weight:
1212                         return (u64)blkcg->weight;
1213                 }
1214                 break;
1215         default:
1216                 BUG();
1217         }
1218         return 0;
1219 }
1220
1221 static int
1222 blkiocg_file_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
1223 {
1224         struct blkio_cgroup *blkcg;
1225         enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1226         int name = BLKIOFILE_ATTR(cft->private);
1227
1228         blkcg = cgroup_to_blkio_cgroup(cgrp);
1229
1230         switch(plid) {
1231         case BLKIO_POLICY_PROP:
1232                 switch(name) {
1233                 case BLKIO_PROP_weight:
1234                         return blkio_weight_write(blkcg, val);
1235                 }
1236                 break;
1237         default:
1238                 BUG();
1239         }
1240
1241         return 0;
1242 }
1243
1244 struct cftype blkio_files[] = {
1245         {
1246                 .name = "weight_device",
1247                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1248                                 BLKIO_PROP_weight_device),
1249                 .read_seq_string = blkiocg_file_read,
1250                 .write_string = blkiocg_file_write,
1251                 .max_write_len = 256,
1252         },
1253         {
1254                 .name = "weight",
1255                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1256                                 BLKIO_PROP_weight),
1257                 .read_u64 = blkiocg_file_read_u64,
1258                 .write_u64 = blkiocg_file_write_u64,
1259         },
1260         {
1261                 .name = "time",
1262                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1263                                 BLKIO_PROP_time),
1264                 .read_map = blkiocg_file_read_map,
1265         },
1266         {
1267                 .name = "sectors",
1268                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1269                                 BLKIO_PROP_sectors),
1270                 .read_map = blkiocg_file_read_map,
1271         },
1272         {
1273                 .name = "io_service_bytes",
1274                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1275                                 BLKIO_PROP_io_service_bytes),
1276                 .read_map = blkiocg_file_read_map,
1277         },
1278         {
1279                 .name = "io_serviced",
1280                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1281                                 BLKIO_PROP_io_serviced),
1282                 .read_map = blkiocg_file_read_map,
1283         },
1284         {
1285                 .name = "io_service_time",
1286                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1287                                 BLKIO_PROP_io_service_time),
1288                 .read_map = blkiocg_file_read_map,
1289         },
1290         {
1291                 .name = "io_wait_time",
1292                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1293                                 BLKIO_PROP_io_wait_time),
1294                 .read_map = blkiocg_file_read_map,
1295         },
1296         {
1297                 .name = "io_merged",
1298                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1299                                 BLKIO_PROP_io_merged),
1300                 .read_map = blkiocg_file_read_map,
1301         },
1302         {
1303                 .name = "io_queued",
1304                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1305                                 BLKIO_PROP_io_queued),
1306                 .read_map = blkiocg_file_read_map,
1307         },
1308         {
1309                 .name = "reset_stats",
1310                 .write_u64 = blkiocg_reset_stats,
1311         },
1312 #ifdef CONFIG_BLK_DEV_THROTTLING
1313         {
1314                 .name = "throttle.read_bps_device",
1315                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1316                                 BLKIO_THROTL_read_bps_device),
1317                 .read_seq_string = blkiocg_file_read,
1318                 .write_string = blkiocg_file_write,
1319                 .max_write_len = 256,
1320         },
1321
1322         {
1323                 .name = "throttle.write_bps_device",
1324                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1325                                 BLKIO_THROTL_write_bps_device),
1326                 .read_seq_string = blkiocg_file_read,
1327                 .write_string = blkiocg_file_write,
1328                 .max_write_len = 256,
1329         },
1330
1331         {
1332                 .name = "throttle.read_iops_device",
1333                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1334                                 BLKIO_THROTL_read_iops_device),
1335                 .read_seq_string = blkiocg_file_read,
1336                 .write_string = blkiocg_file_write,
1337                 .max_write_len = 256,
1338         },
1339
1340         {
1341                 .name = "throttle.write_iops_device",
1342                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1343                                 BLKIO_THROTL_write_iops_device),
1344                 .read_seq_string = blkiocg_file_read,
1345                 .write_string = blkiocg_file_write,
1346                 .max_write_len = 256,
1347         },
1348         {
1349                 .name = "throttle.io_service_bytes",
1350                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1351                                 BLKIO_THROTL_io_service_bytes),
1352                 .read_map = blkiocg_file_read_map,
1353         },
1354         {
1355                 .name = "throttle.io_serviced",
1356                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1357                                 BLKIO_THROTL_io_serviced),
1358                 .read_map = blkiocg_file_read_map,
1359         },
1360 #endif /* CONFIG_BLK_DEV_THROTTLING */
1361
1362 #ifdef CONFIG_DEBUG_BLK_CGROUP
1363         {
1364                 .name = "avg_queue_size",
1365                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1366                                 BLKIO_PROP_avg_queue_size),
1367                 .read_map = blkiocg_file_read_map,
1368         },
1369         {
1370                 .name = "group_wait_time",
1371                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1372                                 BLKIO_PROP_group_wait_time),
1373                 .read_map = blkiocg_file_read_map,
1374         },
1375         {
1376                 .name = "idle_time",
1377                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1378                                 BLKIO_PROP_idle_time),
1379                 .read_map = blkiocg_file_read_map,
1380         },
1381         {
1382                 .name = "empty_time",
1383                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1384                                 BLKIO_PROP_empty_time),
1385                 .read_map = blkiocg_file_read_map,
1386         },
1387         {
1388                 .name = "dequeue",
1389                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1390                                 BLKIO_PROP_dequeue),
1391                 .read_map = blkiocg_file_read_map,
1392         },
1393         {
1394                 .name = "unaccounted_time",
1395                 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1396                                 BLKIO_PROP_unaccounted_time),
1397                 .read_map = blkiocg_file_read_map,
1398         },
1399 #endif
1400 };
1401
1402 static int blkiocg_populate(struct cgroup_subsys *subsys, struct cgroup *cgroup)
1403 {
1404         return cgroup_add_files(cgroup, subsys, blkio_files,
1405                                 ARRAY_SIZE(blkio_files));
1406 }
1407
1408 static void blkiocg_destroy(struct cgroup_subsys *subsys, struct cgroup *cgroup)
1409 {
1410         struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
1411         unsigned long flags;
1412         struct blkio_group *blkg;
1413         void *key;
1414         struct blkio_policy_type *blkiop;
1415         struct blkio_policy_node *pn, *pntmp;
1416
1417         rcu_read_lock();
1418         do {
1419                 spin_lock_irqsave(&blkcg->lock, flags);
1420
1421                 if (hlist_empty(&blkcg->blkg_list)) {
1422                         spin_unlock_irqrestore(&blkcg->lock, flags);
1423                         break;
1424                 }
1425
1426                 blkg = hlist_entry(blkcg->blkg_list.first, struct blkio_group,
1427                                         blkcg_node);
1428                 key = rcu_dereference(blkg->key);
1429                 __blkiocg_del_blkio_group(blkg);
1430
1431                 spin_unlock_irqrestore(&blkcg->lock, flags);
1432
1433                 /*
1434                  * This blkio_group is being unlinked as associated cgroup is
1435                  * going away. Let all the IO controlling policies know about
1436                  * this event.
1437                  */
1438                 spin_lock(&blkio_list_lock);
1439                 list_for_each_entry(blkiop, &blkio_list, list) {
1440                         if (blkiop->plid != blkg->plid)
1441                                 continue;
1442                         blkiop->ops.blkio_unlink_group_fn(key, blkg);
1443                 }
1444                 spin_unlock(&blkio_list_lock);
1445         } while (1);
1446
1447         list_for_each_entry_safe(pn, pntmp, &blkcg->policy_list, node) {
1448                 blkio_policy_delete_node(pn);
1449                 kfree(pn);
1450         }
1451
1452         free_css_id(&blkio_subsys, &blkcg->css);
1453         rcu_read_unlock();
1454         if (blkcg != &blkio_root_cgroup)
1455                 kfree(blkcg);
1456 }
1457
1458 static struct cgroup_subsys_state *
1459 blkiocg_create(struct cgroup_subsys *subsys, struct cgroup *cgroup)
1460 {
1461         struct blkio_cgroup *blkcg;
1462         struct cgroup *parent = cgroup->parent;
1463
1464         if (!parent) {
1465                 blkcg = &blkio_root_cgroup;
1466                 goto done;
1467         }
1468
1469         blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1470         if (!blkcg)
1471                 return ERR_PTR(-ENOMEM);
1472
1473         blkcg->weight = BLKIO_WEIGHT_DEFAULT;
1474 done:
1475         spin_lock_init(&blkcg->lock);
1476         INIT_HLIST_HEAD(&blkcg->blkg_list);
1477
1478         INIT_LIST_HEAD(&blkcg->policy_list);
1479         return &blkcg->css;
1480 }
1481
1482 /*
1483  * We cannot support shared io contexts, as we have no mean to support
1484  * two tasks with the same ioc in two different groups without major rework
1485  * of the main cic data structures.  For now we allow a task to change
1486  * its cgroup only if it's the only owner of its ioc.
1487  */
1488 static int blkiocg_can_attach(struct cgroup_subsys *subsys,
1489                                 struct cgroup *cgroup, struct task_struct *tsk,
1490                                 bool threadgroup)
1491 {
1492         struct io_context *ioc;
1493         int ret = 0;
1494
1495         /* task_lock() is needed to avoid races with exit_io_context() */
1496         task_lock(tsk);
1497         ioc = tsk->io_context;
1498         if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1499                 ret = -EINVAL;
1500         task_unlock(tsk);
1501
1502         return ret;
1503 }
1504
1505 static void blkiocg_attach(struct cgroup_subsys *subsys, struct cgroup *cgroup,
1506                                 struct cgroup *prev, struct task_struct *tsk,
1507                                 bool threadgroup)
1508 {
1509         struct io_context *ioc;
1510
1511         task_lock(tsk);
1512         ioc = tsk->io_context;
1513         if (ioc)
1514                 ioc->cgroup_changed = 1;
1515         task_unlock(tsk);
1516 }
1517
1518 void blkio_policy_register(struct blkio_policy_type *blkiop)
1519 {
1520         spin_lock(&blkio_list_lock);
1521         list_add_tail(&blkiop->list, &blkio_list);
1522         spin_unlock(&blkio_list_lock);
1523 }
1524 EXPORT_SYMBOL_GPL(blkio_policy_register);
1525
1526 void blkio_policy_unregister(struct blkio_policy_type *blkiop)
1527 {
1528         spin_lock(&blkio_list_lock);
1529         list_del_init(&blkiop->list);
1530         spin_unlock(&blkio_list_lock);
1531 }
1532 EXPORT_SYMBOL_GPL(blkio_policy_unregister);
1533
1534 static int __init init_cgroup_blkio(void)
1535 {
1536         return cgroup_load_subsys(&blkio_subsys);
1537 }
1538
1539 static void __exit exit_cgroup_blkio(void)
1540 {
1541         cgroup_unload_subsys(&blkio_subsys);
1542 }
1543
1544 module_init(init_cgroup_blkio);
1545 module_exit(exit_cgroup_blkio);
1546 MODULE_LICENSE("GPL");