drivers/base/memory.c

   1 /*
   2  * Memory subsystem support
   3  *
   4  * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
   5  *            Dave Hansen <haveblue@us.ibm.com>
   6  *
   7  * This file provides the necessary infrastructure to represent
   8  * a SPARSEMEM-memory-model system's physical memory in /sysfs.
   9  * All arch-independent code that assumes MEMORY_HOTPLUG requires
  10  * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/init.h>
  15 #include <linux/topology.h>
  16 #include <linux/capability.h>
  17 #include <linux/device.h>
  18 #include <linux/memory.h>
  19 #include <linux/kobject.h>
  20 #include <linux/memory_hotplug.h>
  21 #include <linux/mm.h>
  22 #include <linux/mutex.h>
  23 #include <linux/stat.h>
  24 #include <linux/slab.h>
  25
  26 #include <linux/atomic.h>
  27 #include <asm/uaccess.h>
  28
  29 static DEFINE_MUTEX(mem_sysfs_mutex);
  30
  31 #define MEMORY_CLASS_NAME       "memory"
  32
  33 static int sections_per_block;
  34
  35 static inline int base_memory_block_id(int section_nr)
  36 {
  37         return section_nr / sections_per_block;
  38 }
  39
  40 static struct bus_type memory_subsys = {
  41         .name = MEMORY_CLASS_NAME,
  42         .dev_name = MEMORY_CLASS_NAME,
  43 };
  44
  45 static BLOCKING_NOTIFIER_HEAD(memory_chain);
  46
  47 int register_memory_notifier(struct notifier_block *nb)
  48 {
  49         return blocking_notifier_chain_register(&memory_chain, nb);
  50 }
  51 EXPORT_SYMBOL(register_memory_notifier);
  52
  53 void unregister_memory_notifier(struct notifier_block *nb)
  54 {
  55         blocking_notifier_chain_unregister(&memory_chain, nb);
  56 }
  57 EXPORT_SYMBOL(unregister_memory_notifier);
  58
  59 static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain);
  60
  61 int register_memory_isolate_notifier(struct notifier_block *nb)
  62 {
  63         return atomic_notifier_chain_register(&memory_isolate_chain, nb);
  64 }
  65 EXPORT_SYMBOL(register_memory_isolate_notifier);
  66
  67 void unregister_memory_isolate_notifier(struct notifier_block *nb)
  68 {
  69         atomic_notifier_chain_unregister(&memory_isolate_chain, nb);
  70 }
  71 EXPORT_SYMBOL(unregister_memory_isolate_notifier);
  72
  73 /*
  74  * register_memory - Setup a sysfs device for a memory block
  75  */
  76 static
  77 int register_memory(struct memory_block *memory)
  78 {
  79         int error;
  80
  81         memory->dev.bus = &memory_subsys;
  82         memory->dev.id = memory->start_section_nr / sections_per_block;
  83
  84         error = device_register(&memory->dev);
  85         return error;
  86 }
  87
  88 static void
  89 unregister_memory(struct memory_block *memory)
  90 {
  91         BUG_ON(memory->dev.bus != &memory_subsys);
  92
  93         /* drop the ref. we got in remove_memory_block() */
  94         kobject_put(&memory->dev.kobj);
  95         device_unregister(&memory->dev);
  96 }
  97
  98 unsigned long __weak memory_block_size_bytes(void)
  99 {
 100         return MIN_MEMORY_BLOCK_SIZE;
 101 }
 102
 103 static unsigned long get_memory_block_size(void)
 104 {
 105         unsigned long block_sz;
 106
 107         block_sz = memory_block_size_bytes();
 108
 109         /* Validate blk_sz is a power of 2 and not less than section size */
 110         if ((block_sz & (block_sz - 1)) || (block_sz < MIN_MEMORY_BLOCK_SIZE)) {
 111                 WARN_ON(1);
 112                 block_sz = MIN_MEMORY_BLOCK_SIZE;
 113         }
 114
 115         return block_sz;
 116 }
 117
 118 /*
 119  * use this as the physical section index that this memsection
 120  * uses.
 121  */
 122
 123 static ssize_t show_mem_start_phys_index(struct device *dev,
 124                         struct device_attribute *attr, char *buf)
 125 {
 126         struct memory_block *mem =
 127                 container_of(dev, struct memory_block, dev);
 128         unsigned long phys_index;
 129
 130         phys_index = mem->start_section_nr / sections_per_block;
 131         return sprintf(buf, "%08lx\n", phys_index);
 132 }
 133
 134 static ssize_t show_mem_end_phys_index(struct device *dev,
 135                         struct device_attribute *attr, char *buf)
 136 {
 137         struct memory_block *mem =
 138                 container_of(dev, struct memory_block, dev);
 139         unsigned long phys_index;
 140
 141         phys_index = mem->end_section_nr / sections_per_block;
 142         return sprintf(buf, "%08lx\n", phys_index);
 143 }
 144
 145 /*
 146  * Show whether the section of memory is likely to be hot-removable
 147  */
 148 static ssize_t show_mem_removable(struct device *dev,
 149                         struct device_attribute *attr, char *buf)
 150 {
 151         unsigned long i, pfn;
 152         int ret = 1;
 153         struct memory_block *mem =
 154                 container_of(dev, struct memory_block, dev);
 155
 156         for (i = 0; i < sections_per_block; i++) {
 157                 if (!present_section_nr(mem->start_section_nr + i))
 158                         continue;
 159                 pfn = section_nr_to_pfn(mem->start_section_nr + i);
 160                 ret &= is_mem_section_removable(pfn, PAGES_PER_SECTION);
 161         }
 162
 163         return sprintf(buf, "%d\n", ret);
 164 }
 165
 166 /*
 167  * online, offline, going offline, etc.
 168  */
 169 static ssize_t show_mem_state(struct device *dev,
 170                         struct device_attribute *attr, char *buf)
 171 {
 172         struct memory_block *mem =
 173                 container_of(dev, struct memory_block, dev);
 174         ssize_t len = 0;
 175
 176         /*
 177          * We can probably put these states in a nice little array
 178          * so that they're not open-coded
 179          */
 180         switch (mem->state) {
 181                 case MEM_ONLINE:
 182                         len = sprintf(buf, "online\n");
 183                         break;
 184                 case MEM_OFFLINE:
 185                         len = sprintf(buf, "offline\n");
 186                         break;
 187                 case MEM_GOING_OFFLINE:
 188                         len = sprintf(buf, "going-offline\n");
 189                         break;
 190                 default:
 191                         len = sprintf(buf, "ERROR-UNKNOWN-%ld\n",
 192                                         mem->state);
 193                         WARN_ON(1);
 194                         break;
 195         }
 196
 197         return len;
 198 }
 199
 200 int memory_notify(unsigned long val, void *v)
 201 {
 202         return blocking_notifier_call_chain(&memory_chain, val, v);
 203 }
 204
 205 int memory_isolate_notify(unsigned long val, void *v)
 206 {
 207         return atomic_notifier_call_chain(&memory_isolate_chain, val, v);
 208 }
 209
 210 /*
 211  * The probe routines leave the pages reserved, just as the bootmem code does.
 212  * Make sure they're still that way.
 213  */
 214 static bool pages_correctly_reserved(unsigned long start_pfn,
 215                                         unsigned long nr_pages)
 216 {
 217         int i, j;
 218         struct page *page;
 219         unsigned long pfn = start_pfn;
 220
 221         /*
 222          * memmap between sections is not contiguous except with
 223          * SPARSEMEM_VMEMMAP. We lookup the page once per section
 224          * and assume memmap is contiguous within each section
 225          */
 226         for (i = 0; i < sections_per_block; i++, pfn += PAGES_PER_SECTION) {
 227                 if (WARN_ON_ONCE(!pfn_valid(pfn)))
 228                         return false;
 229                 page = pfn_to_page(pfn);
 230
 231                 for (j = 0; j < PAGES_PER_SECTION; j++) {
 232                         if (PageReserved(page + j))
 233                                 continue;
 234
 235                         printk(KERN_WARNING "section number %ld page number %d "
 236                                 "not reserved, was it already online?\n",
 237                                 pfn_to_section_nr(pfn), j);
 238
 239                         return false;
 240                 }
 241         }
 242
 243         return true;
 244 }
 245
 246 /*
 247  * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
 248  * OK to have direct references to sparsemem variables in here.
 249  */
 250 static int
 251 memory_block_action(unsigned long phys_index, unsigned long action)
 252 {
 253         unsigned long start_pfn, start_paddr;
 254         unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
 255         struct page *first_page;
 256         int ret;
 257
 258         first_page = pfn_to_page(phys_index << PFN_SECTION_SHIFT);
 259
 260         switch (action) {
 261                 case MEM_ONLINE:
 262                         start_pfn = page_to_pfn(first_page);
 263
 264                         if (!pages_correctly_reserved(start_pfn, nr_pages))
 265                                 return -EBUSY;
 266
 267                         ret = online_pages(start_pfn, nr_pages);
 268                         break;
 269                 case MEM_OFFLINE:
 270                         start_paddr = page_to_pfn(first_page) << PAGE_SHIFT;
 271                         ret = remove_memory(start_paddr,
 272                                             nr_pages << PAGE_SHIFT);
 273                         break;
 274                 default:
 275                         WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: "
 276                              "%ld\n", __func__, phys_index, action, action);
 277                         ret = -EINVAL;
 278         }
 279
 280         return ret;
 281 }
 282
 283 static int memory_block_change_state(struct memory_block *mem,
 284                 unsigned long to_state, unsigned long from_state_req)
 285 {
 286         int ret = 0;
 287
 288         mutex_lock(&mem->state_mutex);
 289
 290         if (mem->state != from_state_req) {
 291                 ret = -EINVAL;
 292                 goto out;
 293         }
 294
 295         if (to_state == MEM_OFFLINE)
 296                 mem->state = MEM_GOING_OFFLINE;
 297
 298         ret = memory_block_action(mem->start_section_nr, to_state);
 299
 300         if (ret)
 301                 mem->state = from_state_req;
 302         else
 303                 mem->state = to_state;
 304
 305 out:
 306         mutex_unlock(&mem->state_mutex);
 307         return ret;
 308 }
 309
 310 static ssize_t
 311 store_mem_state(struct device *dev,
 312                 struct device_attribute *attr, const char *buf, size_t count)
 313 {
 314         struct memory_block *mem;
 315         int ret = -EINVAL;
 316
 317         mem = container_of(dev, struct memory_block, dev);
 318
 319         if (!strncmp(buf, "online", min((int)count, 6)))
 320                 ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE);
 321         else if(!strncmp(buf, "offline", min((int)count, 7)))
 322                 ret = memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE);
 323
 324         if (ret)
 325                 return ret;
 326         return count;
 327 }
 328
 329 /*
 330  * phys_device is a bad name for this.  What I really want
 331  * is a way to differentiate between memory ranges that
 332  * are part of physical devices that constitute
 333  * a complete removable unit or fru.
 334  * i.e. do these ranges belong to the same physical device,
 335  * s.t. if I offline all of these sections I can then
 336  * remove the physical device?
 337  */
 338 static ssize_t show_phys_device(struct device *dev,
 339                                 struct device_attribute *attr, char *buf)
 340 {
 341         struct memory_block *mem =
 342                 container_of(dev, struct memory_block, dev);
 343         return sprintf(buf, "%d\n", mem->phys_device);
 344 }
 345
 346 static DEVICE_ATTR(phys_index, 0444, show_mem_start_phys_index, NULL);
 347 static DEVICE_ATTR(end_phys_index, 0444, show_mem_end_phys_index, NULL);
 348 static DEVICE_ATTR(state, 0644, show_mem_state, store_mem_state);
 349 static DEVICE_ATTR(phys_device, 0444, show_phys_device, NULL);
 350 static DEVICE_ATTR(removable, 0444, show_mem_removable, NULL);
 351
 352 #define mem_create_simple_file(mem, attr_name)  \
 353         device_create_file(&mem->dev, &dev_attr_##attr_name)
 354 #define mem_remove_simple_file(mem, attr_name)  \
 355         device_remove_file(&mem->dev, &dev_attr_##attr_name)
 356
 357 /*
 358  * Block size attribute stuff
 359  */
 360 static ssize_t
 361 print_block_size(struct device *dev, struct device_attribute *attr,
 362                  char *buf)
 363 {
 364         return sprintf(buf, "%lx\n", get_memory_block_size());
 365 }
 366
 367 static DEVICE_ATTR(block_size_bytes, 0444, print_block_size, NULL);
 368
 369 static int block_size_init(void)
 370 {
 371         return device_create_file(memory_subsys.dev_root,
 372                                   &dev_attr_block_size_bytes);
 373 }
 374
 375 /*
 376  * Some architectures will have custom drivers to do this, and
 377  * will not need to do it from userspace.  The fake hot-add code
 378  * as well as ppc64 will do all of their discovery in userspace
 379  * and will require this interface.
 380  */
 381 #ifdef CONFIG_ARCH_MEMORY_PROBE
 382 static ssize_t
 383 memory_probe_store(struct device *dev, struct device_attribute *attr,
 384                    const char *buf, size_t count)
 385 {
 386         u64 phys_addr;
 387         int nid;
 388         int i, ret;
 389         unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;
 390
 391         phys_addr = simple_strtoull(buf, NULL, 0);
 392
 393         if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))
 394                 return -EINVAL;
 395
 396         for (i = 0; i < sections_per_block; i++) {
 397                 nid = memory_add_physaddr_to_nid(phys_addr);
 398                 ret = add_memory(nid, phys_addr,
 399                                  PAGES_PER_SECTION << PAGE_SHIFT);
 400                 if (ret)
 401                         goto out;
 402
 403                 phys_addr += MIN_MEMORY_BLOCK_SIZE;
 404         }
 405
 406         ret = count;
 407 out:
 408         return ret;
 409 }
 410 static DEVICE_ATTR(probe, S_IWUSR, NULL, memory_probe_store);
 411
 412 static int memory_probe_init(void)
 413 {
 414         return device_create_file(memory_subsys.dev_root, &dev_attr_probe);
 415 }
 416 #else
 417 static inline int memory_probe_init(void)
 418 {
 419         return 0;
 420 }
 421 #endif
 422
 423 #ifdef CONFIG_MEMORY_FAILURE
 424 /*
 425  * Support for offlining pages of memory
 426  */
 427
 428 /* Soft offline a page */
 429 static ssize_t
 430 store_soft_offline_page(struct device *dev,
 431                         struct device_attribute *attr,
 432                         const char *buf, size_t count)
 433 {
 434         int ret;
 435         u64 pfn;
 436         if (!capable(CAP_SYS_ADMIN))
 437                 return -EPERM;
 438         if (strict_strtoull(buf, 0, &pfn) < 0)
 439                 return -EINVAL;
 440         pfn >>= PAGE_SHIFT;
 441         if (!pfn_valid(pfn))
 442                 return -ENXIO;
 443         ret = soft_offline_page(pfn_to_page(pfn), 0);
 444         return ret == 0 ? count : ret;
 445 }
 446
 447 /* Forcibly offline a page, including killing processes. */
 448 static ssize_t
 449 store_hard_offline_page(struct device *dev,
 450                         struct device_attribute *attr,
 451                         const char *buf, size_t count)
 452 {
 453         int ret;
 454         u64 pfn;
 455         if (!capable(CAP_SYS_ADMIN))
 456                 return -EPERM;
 457         if (strict_strtoull(buf, 0, &pfn) < 0)
 458                 return -EINVAL;
 459         pfn >>= PAGE_SHIFT;
 460         ret = __memory_failure(pfn, 0, 0);
 461         return ret ? ret : count;
 462 }
 463
 464 static DEVICE_ATTR(soft_offline_page, 0644, NULL, store_soft_offline_page);
 465 static DEVICE_ATTR(hard_offline_page, 0644, NULL, store_hard_offline_page);
 466
 467 static __init int memory_fail_init(void)
 468 {
 469         int err;
 470
 471         err = device_create_file(memory_subsys.dev_root,
 472                                 &dev_attr_soft_offline_page);
 473         if (!err)
 474                 err = device_create_file(memory_subsys.dev_root,
 475                                 &dev_attr_hard_offline_page);
 476         return err;
 477 }
 478 #else
 479 static inline int memory_fail_init(void)
 480 {
 481         return 0;
 482 }
 483 #endif
 484
 485 /*
 486  * Note that phys_device is optional.  It is here to allow for
 487  * differentiation between which *physical* devices each
 488  * section belongs to...
 489  */
 490 int __weak arch_get_memory_phys_device(unsigned long start_pfn)
 491 {
 492         return 0;
 493 }
 494
 495 /*
 496  * A reference for the returned object is held and the reference for the
 497  * hinted object is released.
 498  */
 499 struct memory_block *find_memory_block_hinted(struct mem_section *section,
 500                                               struct memory_block *hint)
 501 {
 502         int block_id = base_memory_block_id(__section_nr(section));
 503         struct device *hintdev = hint ? &hint->dev : NULL;
 504         struct device *dev;
 505
 506         dev = subsys_find_device_by_id(&memory_subsys, block_id, hintdev);
 507         if (hint)
 508                 put_device(&hint->dev);
 509         if (!dev)
 510                 return NULL;
 511         return container_of(dev, struct memory_block, dev);
 512 }
 513
 514 /*
 515  * For now, we have a linear search to go find the appropriate
 516  * memory_block corresponding to a particular phys_index. If
 517  * this gets to be a real problem, we can always use a radix
 518  * tree or something here.
 519  *
 520  * This could be made generic for all device subsystems.
 521  */
 522 struct memory_block *find_memory_block(struct mem_section *section)
 523 {
 524         return find_memory_block_hinted(section, NULL);
 525 }
 526
 527 static int init_memory_block(struct memory_block **memory,
 528                              struct mem_section *section, unsigned long state)
 529 {
 530         struct memory_block *mem;
 531         unsigned long start_pfn;
 532         int scn_nr;
 533         int ret = 0;
 534
 535         mem = kzalloc(sizeof(*mem), GFP_KERNEL);
 536         if (!mem)
 537                 return -ENOMEM;
 538
 539         scn_nr = __section_nr(section);
 540         mem->start_section_nr =
 541                         base_memory_block_id(scn_nr) * sections_per_block;
 542         mem->end_section_nr = mem->start_section_nr + sections_per_block - 1;
 543         mem->state = state;
 544         mem->section_count++;
 545         mutex_init(&mem->state_mutex);
 546         start_pfn = section_nr_to_pfn(mem->start_section_nr);
 547         mem->phys_device = arch_get_memory_phys_device(start_pfn);
 548
 549         ret = register_memory(mem);
 550         if (!ret)
 551                 ret = mem_create_simple_file(mem, phys_index);
 552         if (!ret)
 553                 ret = mem_create_simple_file(mem, end_phys_index);
 554         if (!ret)
 555                 ret = mem_create_simple_file(mem, state);
 556         if (!ret)
 557                 ret = mem_create_simple_file(mem, phys_device);
 558         if (!ret)
 559                 ret = mem_create_simple_file(mem, removable);
 560
 561         *memory = mem;
 562         return ret;
 563 }
 564
 565 static int add_memory_section(int nid, struct mem_section *section,
 566                         unsigned long state, enum mem_add_context context)
 567 {
 568         struct memory_block *mem;
 569         int ret = 0;
 570
 571         mutex_lock(&mem_sysfs_mutex);
 572
 573         mem = find_memory_block(section);
 574         if (mem) {
 575                 mem->section_count++;
 576                 kobject_put(&mem->dev.kobj);
 577         } else
 578                 ret = init_memory_block(&mem, section, state);
 579
 580         if (!ret) {
 581                 if (context == HOTPLUG &&
 582                     mem->section_count == sections_per_block)
 583                         ret = register_mem_sect_under_node(mem, nid);
 584         }
 585
 586         mutex_unlock(&mem_sysfs_mutex);
 587         return ret;
 588 }
 589
 590 int remove_memory_block(unsigned long node_id, struct mem_section *section,
 591                 int phys_device)
 592 {
 593         struct memory_block *mem;
 594
 595         mutex_lock(&mem_sysfs_mutex);
 596         mem = find_memory_block(section);
 597         unregister_mem_sect_under_nodes(mem, __section_nr(section));
 598
 599         mem->section_count--;
 600         if (mem->section_count == 0) {
 601                 mem_remove_simple_file(mem, phys_index);
 602                 mem_remove_simple_file(mem, end_phys_index);
 603                 mem_remove_simple_file(mem, state);
 604                 mem_remove_simple_file(mem, phys_device);
 605                 mem_remove_simple_file(mem, removable);
 606                 unregister_memory(mem);
 607                 kfree(mem);
 608         } else
 609                 kobject_put(&mem->dev.kobj);
 610
 611         mutex_unlock(&mem_sysfs_mutex);
 612         return 0;
 613 }
 614
 615 /*
 616  * need an interface for the VM to add new memory regions,
 617  * but without onlining it.
 618  */
 619 int register_new_memory(int nid, struct mem_section *section)
 620 {
 621         return add_memory_section(nid, section, MEM_OFFLINE, HOTPLUG);
 622 }
 623
 624 int unregister_memory_section(struct mem_section *section)
 625 {
 626         if (!present_section(section))
 627                 return -EINVAL;
 628
 629         return remove_memory_block(0, section, 0);
 630 }
 631
 632 /*
 633  * Initialize the sysfs support for memory devices...
 634  */
 635 int __init memory_dev_init(void)
 636 {
 637         unsigned int i;
 638         int ret;
 639         int err;
 640         unsigned long block_sz;
 641
 642         ret = subsys_system_register(&memory_subsys, NULL);
 643         if (ret)
 644                 goto out;
 645
 646         block_sz = get_memory_block_size();
 647         sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;
 648
 649         /*
 650          * Create entries for memory sections that were found
 651          * during boot and have been initialized
 652          */
 653         for (i = 0; i < NR_MEM_SECTIONS; i++) {
 654                 if (!present_section_nr(i))
 655                         continue;
 656                 err = add_memory_section(0, __nr_to_section(i), MEM_ONLINE,
 657                                          BOOT);
 658                 if (!ret)
 659                         ret = err;
 660         }
 661
 662         err = memory_probe_init();
 663         if (!ret)
 664                 ret = err;
 665         err = memory_fail_init();
 666         if (!ret)
 667                 ret = err;
 668         err = block_size_init();
 669         if (!ret)
 670                 ret = err;
 671 out:
 672         if (ret)
 673                 printk(KERN_ERR "%s() failed: %d\n", __func__, ret);
 674         return ret;
 675 }