mm/vmstat.c

   1 /*
   2  *  linux/mm/vmstat.c
   3  *
   4  *  Manages VM statistics
   5  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   6  *
   7  *  zoned VM statistics
   8  *  Copyright (C) 2006 Silicon Graphics, Inc.,
   9  *              Christoph Lameter <christoph@lameter.com>
  10  */
  11
  12 #include <linux/config.h>
  13 #include <linux/mm.h>
  14 #include <linux/module.h>
  15
  16 void __get_zone_counts(unsigned long *active, unsigned long *inactive,
  17                         unsigned long *free, struct pglist_data *pgdat)
  18 {
  19         struct zone *zones = pgdat->node_zones;
  20         int i;
  21
  22         *active = 0;
  23         *inactive = 0;
  24         *free = 0;
  25         for (i = 0; i < MAX_NR_ZONES; i++) {
  26                 *active += zones[i].nr_active;
  27                 *inactive += zones[i].nr_inactive;
  28                 *free += zones[i].free_pages;
  29         }
  30 }
  31
  32 void get_zone_counts(unsigned long *active,
  33                 unsigned long *inactive, unsigned long *free)
  34 {
  35         struct pglist_data *pgdat;
  36
  37         *active = 0;
  38         *inactive = 0;
  39         *free = 0;
  40         for_each_online_pgdat(pgdat) {
  41                 unsigned long l, m, n;
  42                 __get_zone_counts(&l, &m, &n, pgdat);
  43                 *active += l;
  44                 *inactive += m;
  45                 *free += n;
  46         }
  47 }
  48
  49 #ifdef CONFIG_VM_EVENT_COUNTERS
  50 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
  51 EXPORT_PER_CPU_SYMBOL(vm_event_states);
  52
  53 static void sum_vm_events(unsigned long *ret, cpumask_t *cpumask)
  54 {
  55         int cpu = 0;
  56         int i;
  57
  58         memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
  59
  60         cpu = first_cpu(*cpumask);
  61         while (cpu < NR_CPUS) {
  62                 struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
  63
  64                 cpu = next_cpu(cpu, *cpumask);
  65
  66                 if (cpu < NR_CPUS)
  67                         prefetch(&per_cpu(vm_event_states, cpu));
  68
  69
  70                 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
  71                         ret[i] += this->event[i];
  72         }
  73 }
  74
  75 /*
  76  * Accumulate the vm event counters across all CPUs.
  77  * The result is unavoidably approximate - it can change
  78  * during and after execution of this function.
  79 */
  80 void all_vm_events(unsigned long *ret)
  81 {
  82         sum_vm_events(ret, &cpu_online_map);
  83 }
  84 EXPORT_SYMBOL_GPL(all_vm_events);
  85
  86 #ifdef CONFIG_HOTPLUG
  87 /*
  88  * Fold the foreign cpu events into our own.
  89  *
  90  * This is adding to the events on one processor
  91  * but keeps the global counts constant.
  92  */
  93 void vm_events_fold_cpu(int cpu)
  94 {
  95         struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
  96         int i;
  97
  98         for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
  99                 count_vm_events(i, fold_state->event[i]);
 100                 fold_state->event[i] = 0;
 101         }
 102 }
 103 #endif /* CONFIG_HOTPLUG */
 104
 105 #endif /* CONFIG_VM_EVENT_COUNTERS */
 106
 107 /*
 108  * Manage combined zone based / global counters
 109  *
 110  * vm_stat contains the global counters
 111  */
 112 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
 113 EXPORT_SYMBOL(vm_stat);
 114
 115 #ifdef CONFIG_SMP
 116
 117 #define STAT_THRESHOLD 32
 118
 119 /*
 120  * Determine pointer to currently valid differential byte given a zone and
 121  * the item number.
 122  *
 123  * Preemption must be off
 124  */
 125 static inline s8 *diff_pointer(struct zone *zone, enum zone_stat_item item)
 126 {
 127         return &zone_pcp(zone, smp_processor_id())->vm_stat_diff[item];
 128 }
 129
 130 /*
 131  * For use when we know that interrupts are disabled.
 132  */
 133 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
 134                                 int delta)
 135 {
 136         s8 *p;
 137         long x;
 138
 139         p = diff_pointer(zone, item);
 140         x = delta + *p;
 141
 142         if (unlikely(x > STAT_THRESHOLD || x < -STAT_THRESHOLD)) {
 143                 zone_page_state_add(x, zone, item);
 144                 x = 0;
 145         }
 146
 147         *p = x;
 148 }
 149 EXPORT_SYMBOL(__mod_zone_page_state);
 150
 151 /*
 152  * For an unknown interrupt state
 153  */
 154 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
 155                                         int delta)
 156 {
 157         unsigned long flags;
 158
 159         local_irq_save(flags);
 160         __mod_zone_page_state(zone, item, delta);
 161         local_irq_restore(flags);
 162 }
 163 EXPORT_SYMBOL(mod_zone_page_state);
 164
 165 /*
 166  * Optimized increment and decrement functions.
 167  *
 168  * These are only for a single page and therefore can take a struct page *
 169  * argument instead of struct zone *. This allows the inclusion of the code
 170  * generated for page_zone(page) into the optimized functions.
 171  *
 172  * No overflow check is necessary and therefore the differential can be
 173  * incremented or decremented in place which may allow the compilers to
 174  * generate better code.
 175  *
 176  * The increment or decrement is known and therefore one boundary check can
 177  * be omitted.
 178  *
 179  * Some processors have inc/dec instructions that are atomic vs an interrupt.
 180  * However, the code must first determine the differential location in a zone
 181  * based on the processor number and then inc/dec the counter. There is no
 182  * guarantee without disabling preemption that the processor will not change
 183  * in between and therefore the atomicity vs. interrupt cannot be exploited
 184  * in a useful way here.
 185  */
 186 static void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
 187 {
 188         s8 *p = diff_pointer(zone, item);
 189
 190         (*p)++;
 191
 192         if (unlikely(*p > STAT_THRESHOLD)) {
 193                 zone_page_state_add(*p, zone, item);
 194                 *p = 0;
 195         }
 196 }
 197
 198 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
 199 {
 200         __inc_zone_state(page_zone(page), item);
 201 }
 202 EXPORT_SYMBOL(__inc_zone_page_state);
 203
 204 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
 205 {
 206         struct zone *zone = page_zone(page);
 207         s8 *p = diff_pointer(zone, item);
 208
 209         (*p)--;
 210
 211         if (unlikely(*p < -STAT_THRESHOLD)) {
 212                 zone_page_state_add(*p, zone, item);
 213                 *p = 0;
 214         }
 215 }
 216 EXPORT_SYMBOL(__dec_zone_page_state);
 217
 218 void inc_zone_state(struct zone *zone, enum zone_stat_item item)
 219 {
 220         unsigned long flags;
 221
 222         local_irq_save(flags);
 223         __inc_zone_state(zone, item);
 224         local_irq_restore(flags);
 225 }
 226
 227 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
 228 {
 229         unsigned long flags;
 230         struct zone *zone;
 231
 232         zone = page_zone(page);
 233         local_irq_save(flags);
 234         __inc_zone_state(zone, item);
 235         local_irq_restore(flags);
 236 }
 237 EXPORT_SYMBOL(inc_zone_page_state);
 238
 239 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
 240 {
 241         unsigned long flags;
 242         struct zone *zone;
 243         s8 *p;
 244
 245         zone = page_zone(page);
 246         local_irq_save(flags);
 247         p = diff_pointer(zone, item);
 248
 249         (*p)--;
 250
 251         if (unlikely(*p < -STAT_THRESHOLD)) {
 252                 zone_page_state_add(*p, zone, item);
 253                 *p = 0;
 254         }
 255         local_irq_restore(flags);
 256 }
 257 EXPORT_SYMBOL(dec_zone_page_state);
 258
 259 /*
 260  * Update the zone counters for one cpu.
 261  */
 262 void refresh_cpu_vm_stats(int cpu)
 263 {
 264         struct zone *zone;
 265         int i;
 266         unsigned long flags;
 267
 268         for_each_zone(zone) {
 269                 struct per_cpu_pageset *pcp;
 270
 271                 pcp = zone_pcp(zone, cpu);
 272
 273                 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
 274                         if (pcp->vm_stat_diff[i]) {
 275                                 local_irq_save(flags);
 276                                 zone_page_state_add(pcp->vm_stat_diff[i],
 277                                         zone, i);
 278                                 pcp->vm_stat_diff[i] = 0;
 279                                 local_irq_restore(flags);
 280                         }
 281         }
 282 }
 283
 284 static void __refresh_cpu_vm_stats(void *dummy)
 285 {
 286         refresh_cpu_vm_stats(smp_processor_id());
 287 }
 288
 289 /*
 290  * Consolidate all counters.
 291  *
 292  * Note that the result is less inaccurate but still inaccurate
 293  * if concurrent processes are allowed to run.
 294  */
 295 void refresh_vm_stats(void)
 296 {
 297         on_each_cpu(__refresh_cpu_vm_stats, NULL, 0, 1);
 298 }
 299 EXPORT_SYMBOL(refresh_vm_stats);
 300
 301 #endif
 302
 303 #ifdef CONFIG_NUMA
 304 /*
 305  * zonelist = the list of zones passed to the allocator
 306  * z        = the zone from which the allocation occurred.
 307  *
 308  * Must be called with interrupts disabled.
 309  */
 310 void zone_statistics(struct zonelist *zonelist, struct zone *z)
 311 {
 312         if (z->zone_pgdat == zonelist->zones[0]->zone_pgdat) {
 313                 __inc_zone_state(z, NUMA_HIT);
 314         } else {
 315                 __inc_zone_state(z, NUMA_MISS);
 316                 __inc_zone_state(zonelist->zones[0], NUMA_FOREIGN);
 317         }
 318         if (z->zone_pgdat == NODE_DATA(numa_node_id()))
 319                 __inc_zone_state(z, NUMA_LOCAL);
 320         else
 321                 __inc_zone_state(z, NUMA_OTHER);
 322 }
 323 #endif
 324
 325 #ifdef CONFIG_PROC_FS
 326
 327 #include <linux/seq_file.h>
 328
 329 static void *frag_start(struct seq_file *m, loff_t *pos)
 330 {
 331         pg_data_t *pgdat;
 332         loff_t node = *pos;
 333         for (pgdat = first_online_pgdat();
 334              pgdat && node;
 335              pgdat = next_online_pgdat(pgdat))
 336                 --node;
 337
 338         return pgdat;
 339 }
 340
 341 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
 342 {
 343         pg_data_t *pgdat = (pg_data_t *)arg;
 344
 345         (*pos)++;
 346         return next_online_pgdat(pgdat);
 347 }
 348
 349 static void frag_stop(struct seq_file *m, void *arg)
 350 {
 351 }
 352
 353 /*
 354  * This walks the free areas for each zone.
 355  */
 356 static int frag_show(struct seq_file *m, void *arg)
 357 {
 358         pg_data_t *pgdat = (pg_data_t *)arg;
 359         struct zone *zone;
 360         struct zone *node_zones = pgdat->node_zones;
 361         unsigned long flags;
 362         int order;
 363
 364         for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
 365                 if (!populated_zone(zone))
 366                         continue;
 367
 368                 spin_lock_irqsave(&zone->lock, flags);
 369                 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
 370                 for (order = 0; order < MAX_ORDER; ++order)
 371                         seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
 372                 spin_unlock_irqrestore(&zone->lock, flags);
 373                 seq_putc(m, '\n');
 374         }
 375         return 0;
 376 }
 377
 378 struct seq_operations fragmentation_op = {
 379         .start  = frag_start,
 380         .next   = frag_next,
 381         .stop   = frag_stop,
 382         .show   = frag_show,
 383 };
 384
 385 static char *vmstat_text[] = {
 386         /* Zoned VM counters */
 387         "nr_anon_pages",
 388         "nr_mapped",
 389         "nr_file_pages",
 390         "nr_slab",
 391         "nr_page_table_pages",
 392         "nr_dirty",
 393         "nr_writeback",
 394         "nr_unstable",
 395         "nr_bounce",
 396
 397 #ifdef CONFIG_NUMA
 398         "numa_hit",
 399         "numa_miss",
 400         "numa_foreign",
 401         "numa_interleave",
 402         "numa_local",
 403         "numa_other",
 404 #endif
 405
 406 #ifdef CONFIG_VM_EVENT_COUNTERS
 407         "pgpgin",
 408         "pgpgout",
 409         "pswpin",
 410         "pswpout",
 411
 412         "pgalloc_dma",
 413         "pgalloc_dma32",
 414         "pgalloc_normal",
 415         "pgalloc_high",
 416
 417         "pgfree",
 418         "pgactivate",
 419         "pgdeactivate",
 420
 421         "pgfault",
 422         "pgmajfault",
 423
 424         "pgrefill_dma",
 425         "pgrefill_dma32",
 426         "pgrefill_normal",
 427         "pgrefill_high",
 428
 429         "pgsteal_dma",
 430         "pgsteal_dma32",
 431         "pgsteal_normal",
 432         "pgsteal_high",
 433
 434         "pgscan_kswapd_dma",
 435         "pgscan_kswapd_dma32",
 436         "pgscan_kswapd_normal",
 437         "pgscan_kswapd_high",
 438
 439         "pgscan_direct_dma",
 440         "pgscan_direct_dma32",
 441         "pgscan_direct_normal",
 442         "pgscan_direct_high",
 443
 444         "pginodesteal",
 445         "slabs_scanned",
 446         "kswapd_steal",
 447         "kswapd_inodesteal",
 448         "pageoutrun",
 449         "allocstall",
 450
 451         "pgrotated",
 452 #endif
 453 };
 454
 455 /*
 456  * Output information about zones in @pgdat.
 457  */
 458 static int zoneinfo_show(struct seq_file *m, void *arg)
 459 {
 460         pg_data_t *pgdat = arg;
 461         struct zone *zone;
 462         struct zone *node_zones = pgdat->node_zones;
 463         unsigned long flags;
 464
 465         for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; zone++) {
 466                 int i;
 467
 468                 if (!populated_zone(zone))
 469                         continue;
 470
 471                 spin_lock_irqsave(&zone->lock, flags);
 472                 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
 473                 seq_printf(m,
 474                            "\n  pages free     %lu"
 475                            "\n        min      %lu"
 476                            "\n        low      %lu"
 477                            "\n        high     %lu"
 478                            "\n        active   %lu"
 479                            "\n        inactive %lu"
 480                            "\n        scanned  %lu (a: %lu i: %lu)"
 481                            "\n        spanned  %lu"
 482                            "\n        present  %lu",
 483                            zone->free_pages,
 484                            zone->pages_min,
 485                            zone->pages_low,
 486                            zone->pages_high,
 487                            zone->nr_active,
 488                            zone->nr_inactive,
 489                            zone->pages_scanned,
 490                            zone->nr_scan_active, zone->nr_scan_inactive,
 491                            zone->spanned_pages,
 492                            zone->present_pages);
 493
 494                 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
 495                         seq_printf(m, "\n    %-12s %lu", vmstat_text[i],
 496                                         zone_page_state(zone, i));
 497
 498                 seq_printf(m,
 499                            "\n        protection: (%lu",
 500                            zone->lowmem_reserve[0]);
 501                 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
 502                         seq_printf(m, ", %lu", zone->lowmem_reserve[i]);
 503                 seq_printf(m,
 504                            ")"
 505                            "\n  pagesets");
 506                 for_each_online_cpu(i) {
 507                         struct per_cpu_pageset *pageset;
 508                         int j;
 509
 510                         pageset = zone_pcp(zone, i);
 511                         for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) {
 512                                 if (pageset->pcp[j].count)
 513                                         break;
 514                         }
 515                         if (j == ARRAY_SIZE(pageset->pcp))
 516                                 continue;
 517                         for (j = 0; j < ARRAY_SIZE(pageset->pcp); j++) {
 518                                 seq_printf(m,
 519                                            "\n    cpu: %i pcp: %i"
 520                                            "\n              count: %i"
 521                                            "\n              high:  %i"
 522                                            "\n              batch: %i",
 523                                            i, j,
 524                                            pageset->pcp[j].count,
 525                                            pageset->pcp[j].high,
 526                                            pageset->pcp[j].batch);
 527                         }
 528                 }
 529                 seq_printf(m,
 530                            "\n  all_unreclaimable: %u"
 531                            "\n  prev_priority:     %i"
 532                            "\n  temp_priority:     %i"
 533                            "\n  start_pfn:         %lu",
 534                            zone->all_unreclaimable,
 535                            zone->prev_priority,
 536                            zone->temp_priority,
 537                            zone->zone_start_pfn);
 538                 spin_unlock_irqrestore(&zone->lock, flags);
 539                 seq_putc(m, '\n');
 540         }
 541         return 0;
 542 }
 543
 544 struct seq_operations zoneinfo_op = {
 545         .start  = frag_start, /* iterate over all zones. The same as in
 546                                * fragmentation. */
 547         .next   = frag_next,
 548         .stop   = frag_stop,
 549         .show   = zoneinfo_show,
 550 };
 551
 552 static void *vmstat_start(struct seq_file *m, loff_t *pos)
 553 {
 554         unsigned long *v;
 555 #ifdef CONFIG_VM_EVENT_COUNTERS
 556         unsigned long *e;
 557 #endif
 558         int i;
 559
 560         if (*pos >= ARRAY_SIZE(vmstat_text))
 561                 return NULL;
 562
 563 #ifdef CONFIG_VM_EVENT_COUNTERS
 564         v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long)
 565                         + sizeof(struct vm_event_state), GFP_KERNEL);
 566 #else
 567         v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long),
 568                         GFP_KERNEL);
 569 #endif
 570         m->private = v;
 571         if (!v)
 572                 return ERR_PTR(-ENOMEM);
 573         for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
 574                 v[i] = global_page_state(i);
 575 #ifdef CONFIG_VM_EVENT_COUNTERS
 576         e = v + NR_VM_ZONE_STAT_ITEMS;
 577         all_vm_events(e);
 578         e[PGPGIN] /= 2;         /* sectors -> kbytes */
 579         e[PGPGOUT] /= 2;
 580 #endif
 581         return v + *pos;
 582 }
 583
 584 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
 585 {
 586         (*pos)++;
 587         if (*pos >= ARRAY_SIZE(vmstat_text))
 588                 return NULL;
 589         return (unsigned long *)m->private + *pos;
 590 }
 591
 592 static int vmstat_show(struct seq_file *m, void *arg)
 593 {
 594         unsigned long *l = arg;
 595         unsigned long off = l - (unsigned long *)m->private;
 596
 597         seq_printf(m, "%s %lu\n", vmstat_text[off], *l);
 598         return 0;
 599 }
 600
 601 static void vmstat_stop(struct seq_file *m, void *arg)
 602 {
 603         kfree(m->private);
 604         m->private = NULL;
 605 }
 606
 607 struct seq_operations vmstat_op = {
 608         .start  = vmstat_start,
 609         .next   = vmstat_next,
 610         .stop   = vmstat_stop,
 611         .show   = vmstat_show,
 612 };
 613
 614 #endif /* CONFIG_PROC_FS */
 615