arch/x86/mm/ioremap.c

   1 /*
   2  * Re-map IO memory to kernel address space so that we can access it.
   3  * This is needed for high PCI addresses that aren't mapped in the
   4  * 640k-1MB IO memory area on PC's
   5  *
   6  * (C) Copyright 1995 1996 Linus Torvalds
   7  */
   8
   9 #include <linux/bootmem.h>
  10 #include <linux/init.h>
  11 #include <linux/io.h>
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include <linux/vmalloc.h>
  15 #include <linux/mmiotrace.h>
  16
  17 #include <asm/cacheflush.h>
  18 #include <asm/e820.h>
  19 #include <asm/fixmap.h>
  20 #include <asm/pgtable.h>
  21 #include <asm/tlbflush.h>
  22 #include <asm/pgalloc.h>
  23 #include <asm/pat.h>
  24
  25 #ifdef CONFIG_X86_64
  26
  27 static inline int phys_addr_valid(unsigned long addr)
  28 {
  29         return addr < (1UL << boot_cpu_data.x86_phys_bits);
  30 }
  31
  32 unsigned long __phys_addr(unsigned long x)
  33 {
  34         if (x >= __START_KERNEL_map) {
  35                 x -= __START_KERNEL_map;
  36                 VIRTUAL_BUG_ON(x >= KERNEL_IMAGE_SIZE);
  37                 x += phys_base;
  38         } else {
  39                 VIRTUAL_BUG_ON(x < PAGE_OFFSET);
  40                 x -= PAGE_OFFSET;
  41                 VIRTUAL_BUG_ON(system_state == SYSTEM_BOOTING ? x > MAXMEM :
  42                                         !phys_addr_valid(x));
  43         }
  44         return x;
  45 }
  46 EXPORT_SYMBOL(__phys_addr);
  47
  48 bool __virt_addr_valid(unsigned long x)
  49 {
  50         if (x >= __START_KERNEL_map) {
  51                 x -= __START_KERNEL_map;
  52                 if (x >= KERNEL_IMAGE_SIZE)
  53                         return false;
  54                 x += phys_base;
  55         } else {
  56                 if (x < PAGE_OFFSET)
  57                         return false;
  58                 x -= PAGE_OFFSET;
  59                 if (system_state == SYSTEM_BOOTING ?
  60                                 x > MAXMEM : !phys_addr_valid(x)) {
  61                         return false;
  62                 }
  63         }
  64
  65         return pfn_valid(x >> PAGE_SHIFT);
  66 }
  67 EXPORT_SYMBOL(__virt_addr_valid);
  68
  69 #else
  70
  71 static inline int phys_addr_valid(unsigned long addr)
  72 {
  73         return 1;
  74 }
  75
  76 #ifdef CONFIG_DEBUG_VIRTUAL
  77 unsigned long __phys_addr(unsigned long x)
  78 {
  79         /* VMALLOC_* aren't constants; not available at the boot time */
  80         VIRTUAL_BUG_ON(x < PAGE_OFFSET);
  81         VIRTUAL_BUG_ON(system_state != SYSTEM_BOOTING &&
  82                 is_vmalloc_addr((void *) x));
  83         return x - PAGE_OFFSET;
  84 }
  85 EXPORT_SYMBOL(__phys_addr);
  86 #endif
  87
  88 bool __virt_addr_valid(unsigned long x)
  89 {
  90         if (x < PAGE_OFFSET)
  91                 return false;
  92         if (system_state != SYSTEM_BOOTING && is_vmalloc_addr((void *) x))
  93                 return false;
  94         return pfn_valid((x - PAGE_OFFSET) >> PAGE_SHIFT);
  95 }
  96 EXPORT_SYMBOL(__virt_addr_valid);
  97
  98 #endif
  99
 100 int page_is_ram(unsigned long pagenr)
 101 {
 102         resource_size_t addr, end;
 103         int i;
 104
 105         /*
 106          * A special case is the first 4Kb of memory;
 107          * This is a BIOS owned area, not kernel ram, but generally
 108          * not listed as such in the E820 table.
 109          */
 110         if (pagenr == 0)
 111                 return 0;
 112
 113         /*
 114          * Second special case: Some BIOSen report the PC BIOS
 115          * area (640->1Mb) as ram even though it is not.
 116          */
 117         if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
 118                     pagenr < (BIOS_END >> PAGE_SHIFT))
 119                 return 0;
 120
 121         for (i = 0; i < e820.nr_map; i++) {
 122                 /*
 123                  * Not usable memory:
 124                  */
 125                 if (e820.map[i].type != E820_RAM)
 126                         continue;
 127                 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
 128                 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
 129
 130
 131                 if ((pagenr >= addr) && (pagenr < end))
 132                         return 1;
 133         }
 134         return 0;
 135 }
 136
 137 /*
 138  * Fix up the linear direct mapping of the kernel to avoid cache attribute
 139  * conflicts.
 140  */
 141 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
 142                                unsigned long prot_val)
 143 {
 144         unsigned long nrpages = size >> PAGE_SHIFT;
 145         int err;
 146
 147         switch (prot_val) {
 148         case _PAGE_CACHE_UC:
 149         default:
 150                 err = _set_memory_uc(vaddr, nrpages);
 151                 break;
 152         case _PAGE_CACHE_WC:
 153                 err = _set_memory_wc(vaddr, nrpages);
 154                 break;
 155         case _PAGE_CACHE_WB:
 156                 err = _set_memory_wb(vaddr, nrpages);
 157                 break;
 158         }
 159
 160         return err;
 161 }
 162
 163 /*
 164  * Remap an arbitrary physical address space into the kernel virtual
 165  * address space. Needed when the kernel wants to access high addresses
 166  * directly.
 167  *
 168  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
 169  * have to convert them into an offset in a page-aligned mapping, but the
 170  * caller shouldn't need to know that small detail.
 171  */
 172 static void __iomem *__ioremap_caller(resource_size_t phys_addr,
 173                 unsigned long size, unsigned long prot_val, void *caller)
 174 {
 175         unsigned long pfn, offset, vaddr;
 176         resource_size_t last_addr;
 177         const resource_size_t unaligned_phys_addr = phys_addr;
 178         const unsigned long unaligned_size = size;
 179         struct vm_struct *area;
 180         unsigned long new_prot_val;
 181         pgprot_t prot;
 182         int retval;
 183         void __iomem *ret_addr;
 184
 185         /* Don't allow wraparound or zero size */
 186         last_addr = phys_addr + size - 1;
 187         if (!size || last_addr < phys_addr)
 188                 return NULL;
 189
 190         if (!phys_addr_valid(phys_addr)) {
 191                 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
 192                        (unsigned long long)phys_addr);
 193                 WARN_ON_ONCE(1);
 194                 return NULL;
 195         }
 196
 197         /*
 198          * Don't remap the low PCI/ISA area, it's always mapped..
 199          */
 200         if (is_ISA_range(phys_addr, last_addr))
 201                 return (__force void __iomem *)phys_to_virt(phys_addr);
 202
 203         /*
 204          * Check if the request spans more than any BAR in the iomem resource
 205          * tree.
 206          */
 207         WARN_ONCE(iomem_map_sanity_check(phys_addr, size),
 208                   KERN_INFO "Info: mapping multiple BARs. Your kernel is fine.");
 209
 210         /*
 211          * Don't allow anybody to remap normal RAM that we're using..
 212          */
 213         for (pfn = phys_addr >> PAGE_SHIFT;
 214                                 (pfn << PAGE_SHIFT) < (last_addr & PAGE_MASK);
 215                                 pfn++) {
 216
 217                 int is_ram = page_is_ram(pfn);
 218
 219                 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
 220                         return NULL;
 221                 WARN_ON_ONCE(is_ram);
 222         }
 223
 224         /*
 225          * Mappings have to be page-aligned
 226          */
 227         offset = phys_addr & ~PAGE_MASK;
 228         phys_addr &= PAGE_MASK;
 229         size = PAGE_ALIGN(last_addr+1) - phys_addr;
 230
 231         retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
 232                                                 prot_val, &new_prot_val);
 233         if (retval) {
 234                 pr_debug("Warning: reserve_memtype returned %d\n", retval);
 235                 return NULL;
 236         }
 237
 238         if (prot_val != new_prot_val) {
 239                 /*
 240                  * Do not fallback to certain memory types with certain
 241                  * requested type:
 242                  * - request is uc-, return cannot be write-back
 243                  * - request is uc-, return cannot be write-combine
 244                  * - request is write-combine, return cannot be write-back
 245                  */
 246                 if ((prot_val == _PAGE_CACHE_UC_MINUS &&
 247                      (new_prot_val == _PAGE_CACHE_WB ||
 248                       new_prot_val == _PAGE_CACHE_WC)) ||
 249                     (prot_val == _PAGE_CACHE_WC &&
 250                      new_prot_val == _PAGE_CACHE_WB)) {
 251                         pr_debug(
 252                 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
 253                                 (unsigned long long)phys_addr,
 254                                 (unsigned long long)(phys_addr + size),
 255                                 prot_val, new_prot_val);
 256                         free_memtype(phys_addr, phys_addr + size);
 257                         return NULL;
 258                 }
 259                 prot_val = new_prot_val;
 260         }
 261
 262         switch (prot_val) {
 263         case _PAGE_CACHE_UC:
 264         default:
 265                 prot = PAGE_KERNEL_IO_NOCACHE;
 266                 break;
 267         case _PAGE_CACHE_UC_MINUS:
 268                 prot = PAGE_KERNEL_IO_UC_MINUS;
 269                 break;
 270         case _PAGE_CACHE_WC:
 271                 prot = PAGE_KERNEL_IO_WC;
 272                 break;
 273         case _PAGE_CACHE_WB:
 274                 prot = PAGE_KERNEL_IO;
 275                 break;
 276         }
 277
 278         /*
 279          * Ok, go for it..
 280          */
 281         area = get_vm_area_caller(size, VM_IOREMAP, caller);
 282         if (!area)
 283                 return NULL;
 284         area->phys_addr = phys_addr;
 285         vaddr = (unsigned long) area->addr;
 286         if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
 287                 free_memtype(phys_addr, phys_addr + size);
 288                 free_vm_area(area);
 289                 return NULL;
 290         }
 291
 292         if (ioremap_change_attr(vaddr, size, prot_val) < 0) {
 293                 free_memtype(phys_addr, phys_addr + size);
 294                 vunmap(area->addr);
 295                 return NULL;
 296         }
 297
 298         ret_addr = (void __iomem *) (vaddr + offset);
 299         mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
 300
 301         return ret_addr;
 302 }
 303
 304 /**
 305  * ioremap_nocache     -   map bus memory into CPU space
 306  * @offset:    bus address of the memory
 307  * @size:      size of the resource to map
 308  *
 309  * ioremap_nocache performs a platform specific sequence of operations to
 310  * make bus memory CPU accessible via the readb/readw/readl/writeb/
 311  * writew/writel functions and the other mmio helpers. The returned
 312  * address is not guaranteed to be usable directly as a virtual
 313  * address.
 314  *
 315  * This version of ioremap ensures that the memory is marked uncachable
 316  * on the CPU as well as honouring existing caching rules from things like
 317  * the PCI bus. Note that there are other caches and buffers on many
 318  * busses. In particular driver authors should read up on PCI writes
 319  *
 320  * It's useful if some control registers are in such an area and
 321  * write combining or read caching is not desirable:
 322  *
 323  * Must be freed with iounmap.
 324  */
 325 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
 326 {
 327         /*
 328          * Ideally, this should be:
 329          *      pat_enabled ? _PAGE_CACHE_UC : _PAGE_CACHE_UC_MINUS;
 330          *
 331          * Till we fix all X drivers to use ioremap_wc(), we will use
 332          * UC MINUS.
 333          */
 334         unsigned long val = _PAGE_CACHE_UC_MINUS;
 335
 336         return __ioremap_caller(phys_addr, size, val,
 337                                 __builtin_return_address(0));
 338 }
 339 EXPORT_SYMBOL(ioremap_nocache);
 340
 341 /**
 342  * ioremap_wc   -       map memory into CPU space write combined
 343  * @offset:     bus address of the memory
 344  * @size:       size of the resource to map
 345  *
 346  * This version of ioremap ensures that the memory is marked write combining.
 347  * Write combining allows faster writes to some hardware devices.
 348  *
 349  * Must be freed with iounmap.
 350  */
 351 void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
 352 {
 353         if (pat_enabled)
 354                 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WC,
 355                                         __builtin_return_address(0));
 356         else
 357                 return ioremap_nocache(phys_addr, size);
 358 }
 359 EXPORT_SYMBOL(ioremap_wc);
 360
 361 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
 362 {
 363         return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WB,
 364                                 __builtin_return_address(0));
 365 }
 366 EXPORT_SYMBOL(ioremap_cache);
 367
 368 static void __iomem *ioremap_default(resource_size_t phys_addr,
 369                                         unsigned long size)
 370 {
 371         unsigned long flags;
 372         void __iomem *ret;
 373         int err;
 374
 375         /*
 376          * - WB for WB-able memory and no other conflicting mappings
 377          * - UC_MINUS for non-WB-able memory with no other conflicting mappings
 378          * - Inherit from confliting mappings otherwise
 379          */
 380         err = reserve_memtype(phys_addr, phys_addr + size, -1, &flags);
 381         if (err < 0)
 382                 return NULL;
 383
 384         ret = __ioremap_caller(phys_addr, size, flags,
 385                                __builtin_return_address(0));
 386
 387         free_memtype(phys_addr, phys_addr + size);
 388         return ret;
 389 }
 390
 391 void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
 392                                 unsigned long prot_val)
 393 {
 394         return __ioremap_caller(phys_addr, size, (prot_val & _PAGE_CACHE_MASK),
 395                                 __builtin_return_address(0));
 396 }
 397 EXPORT_SYMBOL(ioremap_prot);
 398
 399 /**
 400  * iounmap - Free a IO remapping
 401  * @addr: virtual address from ioremap_*
 402  *
 403  * Caller must ensure there is only one unmapping for the same pointer.
 404  */
 405 void iounmap(volatile void __iomem *addr)
 406 {
 407         struct vm_struct *p, *o;
 408
 409         if ((void __force *)addr <= high_memory)
 410                 return;
 411
 412         /*
 413          * __ioremap special-cases the PCI/ISA range by not instantiating a
 414          * vm_area and by simply returning an address into the kernel mapping
 415          * of ISA space.   So handle that here.
 416          */
 417         if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
 418             (void __force *)addr < phys_to_virt(ISA_END_ADDRESS))
 419                 return;
 420
 421         addr = (volatile void __iomem *)
 422                 (PAGE_MASK & (unsigned long __force)addr);
 423
 424         mmiotrace_iounmap(addr);
 425
 426         /* Use the vm area unlocked, assuming the caller
 427            ensures there isn't another iounmap for the same address
 428            in parallel. Reuse of the virtual address is prevented by
 429            leaving it in the global lists until we're done with it.
 430            cpa takes care of the direct mappings. */
 431         read_lock(&vmlist_lock);
 432         for (p = vmlist; p; p = p->next) {
 433                 if (p->addr == (void __force *)addr)
 434                         break;
 435         }
 436         read_unlock(&vmlist_lock);
 437
 438         if (!p) {
 439                 printk(KERN_ERR "iounmap: bad address %p\n", addr);
 440                 dump_stack();
 441                 return;
 442         }
 443
 444         free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
 445
 446         /* Finally remove it */
 447         o = remove_vm_area((void __force *)addr);
 448         BUG_ON(p != o || o == NULL);
 449         kfree(p);
 450 }
 451 EXPORT_SYMBOL(iounmap);
 452
 453 /*
 454  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
 455  * access
 456  */
 457 void *xlate_dev_mem_ptr(unsigned long phys)
 458 {
 459         void *addr;
 460         unsigned long start = phys & PAGE_MASK;
 461
 462         /* If page is RAM, we can use __va. Otherwise ioremap and unmap. */
 463         if (page_is_ram(start >> PAGE_SHIFT))
 464                 return __va(phys);
 465
 466         addr = (void __force *)ioremap_default(start, PAGE_SIZE);
 467         if (addr)
 468                 addr = (void *)((unsigned long)addr | (phys & ~PAGE_MASK));
 469
 470         return addr;
 471 }
 472
 473 void unxlate_dev_mem_ptr(unsigned long phys, void *addr)
 474 {
 475         if (page_is_ram(phys >> PAGE_SHIFT))
 476                 return;
 477
 478         iounmap((void __iomem *)((unsigned long)addr & PAGE_MASK));
 479         return;
 480 }
 481
 482 static int __initdata early_ioremap_debug;
 483
 484 static int __init early_ioremap_debug_setup(char *str)
 485 {
 486         early_ioremap_debug = 1;
 487
 488         return 0;
 489 }
 490 early_param("early_ioremap_debug", early_ioremap_debug_setup);
 491
 492 static __initdata int after_paging_init;
 493 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
 494
 495 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
 496 {
 497         /* Don't assume we're using swapper_pg_dir at this point */
 498         pgd_t *base = __va(read_cr3());
 499         pgd_t *pgd = &base[pgd_index(addr)];
 500         pud_t *pud = pud_offset(pgd, addr);
 501         pmd_t *pmd = pmd_offset(pud, addr);
 502
 503         return pmd;
 504 }
 505
 506 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
 507 {
 508         return &bm_pte[pte_index(addr)];
 509 }
 510
 511 void __init early_ioremap_init(void)
 512 {
 513         pmd_t *pmd;
 514
 515         if (early_ioremap_debug)
 516                 printk(KERN_INFO "early_ioremap_init()\n");
 517
 518         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 519         memset(bm_pte, 0, sizeof(bm_pte));
 520         pmd_populate_kernel(&init_mm, pmd, bm_pte);
 521
 522         /*
 523          * The boot-ioremap range spans multiple pmds, for which
 524          * we are not prepared:
 525          */
 526         if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
 527                 WARN_ON(1);
 528                 printk(KERN_WARNING "pmd %p != %p\n",
 529                        pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
 530                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
 531                         fix_to_virt(FIX_BTMAP_BEGIN));
 532                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
 533                         fix_to_virt(FIX_BTMAP_END));
 534
 535                 printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
 536                 printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
 537                        FIX_BTMAP_BEGIN);
 538         }
 539 }
 540
 541 void __init early_ioremap_reset(void)
 542 {
 543         after_paging_init = 1;
 544 }
 545
 546 static void __init __early_set_fixmap(enum fixed_addresses idx,
 547                                    unsigned long phys, pgprot_t flags)
 548 {
 549         unsigned long addr = __fix_to_virt(idx);
 550         pte_t *pte;
 551
 552         if (idx >= __end_of_fixed_addresses) {
 553                 BUG();
 554                 return;
 555         }
 556         pte = early_ioremap_pte(addr);
 557
 558         if (pgprot_val(flags))
 559                 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
 560         else
 561                 pte_clear(&init_mm, addr, pte);
 562         __flush_tlb_one(addr);
 563 }
 564
 565 static inline void __init early_set_fixmap(enum fixed_addresses idx,
 566                                            unsigned long phys, pgprot_t prot)
 567 {
 568         if (after_paging_init)
 569                 __set_fixmap(idx, phys, prot);
 570         else
 571                 __early_set_fixmap(idx, phys, prot);
 572 }
 573
 574 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
 575 {
 576         if (after_paging_init)
 577                 clear_fixmap(idx);
 578         else
 579                 __early_set_fixmap(idx, 0, __pgprot(0));
 580 }
 581
 582 static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
 583 static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
 584 static int __init check_early_ioremap_leak(void)
 585 {
 586         int count = 0;
 587         int i;
 588
 589         for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
 590                 if (prev_map[i])
 591                         count++;
 592
 593         if (!count)
 594                 return 0;
 595         WARN(1, KERN_WARNING
 596                "Debug warning: early ioremap leak of %d areas detected.\n",
 597                 count);
 598         printk(KERN_WARNING
 599                 "please boot with early_ioremap_debug and report the dmesg.\n");
 600
 601         return 1;
 602 }
 603 late_initcall(check_early_ioremap_leak);
 604
 605 static void __init __iomem *__early_ioremap(unsigned long phys_addr, unsigned long size, pgprot_t prot)
 606 {
 607         unsigned long offset, last_addr;
 608         unsigned int nrpages;
 609         enum fixed_addresses idx0, idx;
 610         int i, slot;
 611
 612         WARN_ON(system_state != SYSTEM_BOOTING);
 613
 614         slot = -1;
 615         for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
 616                 if (!prev_map[i]) {
 617                         slot = i;
 618                         break;
 619                 }
 620         }
 621
 622         if (slot < 0) {
 623                 printk(KERN_INFO "early_iomap(%08lx, %08lx) not found slot\n",
 624                          phys_addr, size);
 625                 WARN_ON(1);
 626                 return NULL;
 627         }
 628
 629         if (early_ioremap_debug) {
 630                 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
 631                        phys_addr, size, slot);
 632                 dump_stack();
 633         }
 634
 635         /* Don't allow wraparound or zero size */
 636         last_addr = phys_addr + size - 1;
 637         if (!size || last_addr < phys_addr) {
 638                 WARN_ON(1);
 639                 return NULL;
 640         }
 641
 642         prev_size[slot] = size;
 643         /*
 644          * Mappings have to be page-aligned
 645          */
 646         offset = phys_addr & ~PAGE_MASK;
 647         phys_addr &= PAGE_MASK;
 648         size = PAGE_ALIGN(last_addr + 1) - phys_addr;
 649
 650         /*
 651          * Mappings have to fit in the FIX_BTMAP area.
 652          */
 653         nrpages = size >> PAGE_SHIFT;
 654         if (nrpages > NR_FIX_BTMAPS) {
 655                 WARN_ON(1);
 656                 return NULL;
 657         }
 658
 659         /*
 660          * Ok, go for it..
 661          */
 662         idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
 663         idx = idx0;
 664         while (nrpages > 0) {
 665                 early_set_fixmap(idx, phys_addr, prot);
 666                 phys_addr += PAGE_SIZE;
 667                 --idx;
 668                 --nrpages;
 669         }
 670         if (early_ioremap_debug)
 671                 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
 672
 673         prev_map[slot] = (void __iomem *)(offset + fix_to_virt(idx0));
 674         return prev_map[slot];
 675 }
 676
 677 /* Remap an IO device */
 678 void __init __iomem *early_ioremap(unsigned long phys_addr, unsigned long size)
 679 {
 680         return __early_ioremap(phys_addr, size, PAGE_KERNEL_IO);
 681 }
 682
 683 /* Remap memory */
 684 void __init __iomem *early_memremap(unsigned long phys_addr, unsigned long size)
 685 {
 686         return __early_ioremap(phys_addr, size, PAGE_KERNEL);
 687 }
 688
 689 void __init early_iounmap(void __iomem *addr, unsigned long size)
 690 {
 691         unsigned long virt_addr;
 692         unsigned long offset;
 693         unsigned int nrpages;
 694         enum fixed_addresses idx;
 695         int i, slot;
 696
 697         slot = -1;
 698         for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
 699                 if (prev_map[i] == addr) {
 700                         slot = i;
 701                         break;
 702                 }
 703         }
 704
 705         if (slot < 0) {
 706                 printk(KERN_INFO "early_iounmap(%p, %08lx) not found slot\n",
 707                          addr, size);
 708                 WARN_ON(1);
 709                 return;
 710         }
 711
 712         if (prev_size[slot] != size) {
 713                 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n",
 714                          addr, size, slot, prev_size[slot]);
 715                 WARN_ON(1);
 716                 return;
 717         }
 718
 719         if (early_ioremap_debug) {
 720                 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
 721                        size, slot);
 722                 dump_stack();
 723         }
 724
 725         virt_addr = (unsigned long)addr;
 726         if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
 727                 WARN_ON(1);
 728                 return;
 729         }
 730         offset = virt_addr & ~PAGE_MASK;
 731         nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
 732
 733         idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
 734         while (nrpages > 0) {
 735                 early_clear_fixmap(idx);
 736                 --idx;
 737                 --nrpages;
 738         }
 739         prev_map[slot] = NULL;
 740 }
 741
 742 void __this_fixmap_does_not_exist(void)
 743 {
 744         WARN_ON(1);
 745 }