arch/um/kernel/tlb.c

   1 /*
   2  * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
   3  * Licensed under the GPL
   4  */
   5
   6 #include "linux/mm.h"
   7 #include "asm/pgtable.h"
   8 #include "asm/tlbflush.h"
   9 #include "as-layout.h"
  10 #include "mem_user.h"
  11 #include "os.h"
  12 #include "skas.h"
  13 #include "tlb.h"
  14
  15 static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
  16                     unsigned int prot, struct host_vm_op *ops, int *index,
  17                     int last_filled, union mm_context *mmu, void **flush,
  18                     int (*do_ops)(union mm_context *, struct host_vm_op *,
  19                                   int, int, void **))
  20 {
  21         __u64 offset;
  22         struct host_vm_op *last;
  23         int fd, ret = 0;
  24
  25         fd = phys_mapping(phys, &offset);
  26         if (*index != -1) {
  27                 last = &ops[*index];
  28                 if ((last->type == MMAP) &&
  29                    (last->u.mmap.addr + last->u.mmap.len == virt) &&
  30                    (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
  31                    (last->u.mmap.offset + last->u.mmap.len == offset)) {
  32                         last->u.mmap.len += len;
  33                         return 0;
  34                 }
  35         }
  36
  37         if (*index == last_filled) {
  38                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
  39                 *index = -1;
  40         }
  41
  42         ops[++*index] = ((struct host_vm_op) { .type    = MMAP,
  43                                                 .u = { .mmap = {
  44                                                        .addr    = virt,
  45                                                        .len     = len,
  46                                                        .prot    = prot,
  47                                                        .fd      = fd,
  48                                                        .offset  = offset }
  49                            } });
  50         return ret;
  51 }
  52
  53 static int add_munmap(unsigned long addr, unsigned long len,
  54                       struct host_vm_op *ops, int *index, int last_filled,
  55                       union mm_context *mmu, void **flush,
  56                       int (*do_ops)(union mm_context *, struct host_vm_op *,
  57                                     int, int, void **))
  58 {
  59         struct host_vm_op *last;
  60         int ret = 0;
  61
  62         if (*index != -1) {
  63                 last = &ops[*index];
  64                 if ((last->type == MUNMAP) &&
  65                    (last->u.munmap.addr + last->u.mmap.len == addr)) {
  66                         last->u.munmap.len += len;
  67                         return 0;
  68                 }
  69         }
  70
  71         if (*index == last_filled) {
  72                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
  73                 *index = -1;
  74         }
  75
  76         ops[++*index] = ((struct host_vm_op) { .type    = MUNMAP,
  77                                                .u = { .munmap = {
  78                                                         .addr   = addr,
  79                                                         .len    = len } } });
  80         return ret;
  81 }
  82
  83 static int add_mprotect(unsigned long addr, unsigned long len,
  84                         unsigned int prot, struct host_vm_op *ops, int *index,
  85                         int last_filled, union mm_context *mmu, void **flush,
  86                         int (*do_ops)(union mm_context *, struct host_vm_op *,
  87                                       int, int, void **))
  88 {
  89         struct host_vm_op *last;
  90         int ret = 0;
  91
  92         if (*index != -1) {
  93                 last = &ops[*index];
  94                 if ((last->type == MPROTECT) &&
  95                    (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
  96                    (last->u.mprotect.prot == prot)) {
  97                         last->u.mprotect.len += len;
  98                         return 0;
  99                 }
 100         }
 101
 102         if (*index == last_filled) {
 103                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
 104                 *index = -1;
 105         }
 106
 107         ops[++*index] = ((struct host_vm_op) { .type    = MPROTECT,
 108                                                .u = { .mprotect = {
 109                                                        .addr    = addr,
 110                                                        .len     = len,
 111                                                        .prot    = prot } } });
 112         return ret;
 113 }
 114
 115 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
 116
 117 static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
 118                                    unsigned long end, struct host_vm_op *ops,
 119                                    int last_op, int *op_index, int force,
 120                                    union mm_context *mmu, void **flush,
 121                                    int (*do_ops)(union mm_context *,
 122                                                  struct host_vm_op *, int, int,
 123                                                  void **))
 124 {
 125         pte_t *pte;
 126         int r, w, x, prot, ret = 0;
 127
 128         pte = pte_offset_kernel(pmd, addr);
 129         do {
 130                 r = pte_read(*pte);
 131                 w = pte_write(*pte);
 132                 x = pte_exec(*pte);
 133                 if (!pte_young(*pte)) {
 134                         r = 0;
 135                         w = 0;
 136                 } else if (!pte_dirty(*pte)) {
 137                         w = 0;
 138                 }
 139                 prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
 140                         (x ? UM_PROT_EXEC : 0));
 141                 if (force || pte_newpage(*pte)) {
 142                         if (pte_present(*pte))
 143                                 ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
 144                                                PAGE_SIZE, prot, ops, op_index,
 145                                                last_op, mmu, flush, do_ops);
 146                         else ret = add_munmap(addr, PAGE_SIZE, ops, op_index,
 147                                               last_op, mmu, flush, do_ops);
 148                 }
 149                 else if (pte_newprot(*pte))
 150                         ret = add_mprotect(addr, PAGE_SIZE, prot, ops, op_index,
 151                                            last_op, mmu, flush, do_ops);
 152                 *pte = pte_mkuptodate(*pte);
 153         } while (pte++, addr += PAGE_SIZE, ((addr != end) && !ret));
 154         return ret;
 155 }
 156
 157 static inline int update_pmd_range(pud_t *pud, unsigned long addr,
 158                                    unsigned long end, struct host_vm_op *ops,
 159                                    int last_op, int *op_index, int force,
 160                                    union mm_context *mmu, void **flush,
 161                                    int (*do_ops)(union mm_context *,
 162                                                  struct host_vm_op *, int, int,
 163                                                  void **))
 164 {
 165         pmd_t *pmd;
 166         unsigned long next;
 167         int ret = 0;
 168
 169         pmd = pmd_offset(pud, addr);
 170         do {
 171                 next = pmd_addr_end(addr, end);
 172                 if (!pmd_present(*pmd)) {
 173                         if (force || pmd_newpage(*pmd)) {
 174                                 ret = add_munmap(addr, next - addr, ops,
 175                                                  op_index, last_op, mmu,
 176                                                  flush, do_ops);
 177                                 pmd_mkuptodate(*pmd);
 178                         }
 179                 }
 180                 else ret = update_pte_range(pmd, addr, next, ops, last_op,
 181                                             op_index, force, mmu, flush,
 182                                             do_ops);
 183         } while (pmd++, addr = next, ((addr != end) && !ret));
 184         return ret;
 185 }
 186
 187 static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
 188                                    unsigned long end, struct host_vm_op *ops,
 189                                    int last_op, int *op_index, int force,
 190                                    union mm_context *mmu, void **flush,
 191                                    int (*do_ops)(union mm_context *,
 192                                                  struct host_vm_op *, int, int,
 193                                                  void **))
 194 {
 195         pud_t *pud;
 196         unsigned long next;
 197         int ret = 0;
 198
 199         pud = pud_offset(pgd, addr);
 200         do {
 201                 next = pud_addr_end(addr, end);
 202                 if (!pud_present(*pud)) {
 203                         if (force || pud_newpage(*pud)) {
 204                                 ret = add_munmap(addr, next - addr, ops,
 205                                                  op_index, last_op, mmu,
 206                                                  flush, do_ops);
 207                                 pud_mkuptodate(*pud);
 208                         }
 209                 }
 210                 else ret = update_pmd_range(pud, addr, next, ops, last_op,
 211                                             op_index, force, mmu, flush,
 212                                             do_ops);
 213         } while (pud++, addr = next, ((addr != end) && !ret));
 214         return ret;
 215 }
 216
 217 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
 218                       unsigned long end_addr, int force,
 219                       int (*do_ops)(union mm_context *, struct host_vm_op *,
 220                                     int, int, void **))
 221 {
 222         pgd_t *pgd;
 223         union mm_context *mmu = &mm->context;
 224         struct host_vm_op ops[1];
 225         unsigned long addr = start_addr, next;
 226         int ret = 0, last_op = ARRAY_SIZE(ops) - 1, op_index = -1;
 227         void *flush = NULL;
 228
 229         ops[0].type = NONE;
 230         pgd = pgd_offset(mm, addr);
 231         do {
 232                 next = pgd_addr_end(addr, end_addr);
 233                 if (!pgd_present(*pgd)) {
 234                         if (force || pgd_newpage(*pgd)) {
 235                                 ret = add_munmap(addr, next - addr, ops,
 236                                                  &op_index, last_op, mmu,
 237                                                  &flush, do_ops);
 238                                 pgd_mkuptodate(*pgd);
 239                         }
 240                 }
 241                 else ret = update_pud_range(pgd, addr, next, ops, last_op,
 242                                             &op_index, force, mmu, &flush,
 243                                             do_ops);
 244         } while (pgd++, addr = next, ((addr != end_addr) && !ret));
 245
 246         if (!ret)
 247                 ret = (*do_ops)(mmu, ops, op_index, 1, &flush);
 248
 249         /* This is not an else because ret is modified above */
 250         if (ret) {
 251                 printk(KERN_ERR "fix_range_common: failed, killing current "
 252                        "process\n");
 253                 force_sig(SIGKILL, current);
 254         }
 255 }
 256
 257 int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
 258 {
 259         struct mm_struct *mm;
 260         pgd_t *pgd;
 261         pud_t *pud;
 262         pmd_t *pmd;
 263         pte_t *pte;
 264         unsigned long addr, last;
 265         int updated = 0, err;
 266
 267         mm = &init_mm;
 268         for (addr = start; addr < end;) {
 269                 pgd = pgd_offset(mm, addr);
 270                 if (!pgd_present(*pgd)) {
 271                         last = ADD_ROUND(addr, PGDIR_SIZE);
 272                         if (last > end)
 273                                 last = end;
 274                         if (pgd_newpage(*pgd)) {
 275                                 updated = 1;
 276                                 err = os_unmap_memory((void *) addr,
 277                                                       last - addr);
 278                                 if (err < 0)
 279                                         panic("munmap failed, errno = %d\n",
 280                                               -err);
 281                         }
 282                         addr = last;
 283                         continue;
 284                 }
 285
 286                 pud = pud_offset(pgd, addr);
 287                 if (!pud_present(*pud)) {
 288                         last = ADD_ROUND(addr, PUD_SIZE);
 289                         if (last > end)
 290                                 last = end;
 291                         if (pud_newpage(*pud)) {
 292                                 updated = 1;
 293                                 err = os_unmap_memory((void *) addr,
 294                                                       last - addr);
 295                                 if (err < 0)
 296                                         panic("munmap failed, errno = %d\n",
 297                                               -err);
 298                         }
 299                         addr = last;
 300                         continue;
 301                 }
 302
 303                 pmd = pmd_offset(pud, addr);
 304                 if (!pmd_present(*pmd)) {
 305                         last = ADD_ROUND(addr, PMD_SIZE);
 306                         if (last > end)
 307                                 last = end;
 308                         if (pmd_newpage(*pmd)) {
 309                                 updated = 1;
 310                                 err = os_unmap_memory((void *) addr,
 311                                                       last - addr);
 312                                 if (err < 0)
 313                                         panic("munmap failed, errno = %d\n",
 314                                               -err);
 315                         }
 316                         addr = last;
 317                         continue;
 318                 }
 319
 320                 pte = pte_offset_kernel(pmd, addr);
 321                 if (!pte_present(*pte) || pte_newpage(*pte)) {
 322                         updated = 1;
 323                         err = os_unmap_memory((void *) addr,
 324                                               PAGE_SIZE);
 325                         if (err < 0)
 326                                 panic("munmap failed, errno = %d\n",
 327                                       -err);
 328                         if (pte_present(*pte))
 329                                 map_memory(addr,
 330                                            pte_val(*pte) & PAGE_MASK,
 331                                            PAGE_SIZE, 1, 1, 1);
 332                 }
 333                 else if (pte_newprot(*pte)) {
 334                         updated = 1;
 335                         os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
 336                 }
 337                 addr += PAGE_SIZE;
 338         }
 339         return updated;
 340 }
 341
 342 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
 343 {
 344         pgd_t *pgd;
 345         pud_t *pud;
 346         pmd_t *pmd;
 347         pte_t *pte;
 348         struct mm_struct *mm = vma->vm_mm;
 349         void *flush = NULL;
 350         int r, w, x, prot, err = 0;
 351         struct mm_id *mm_id;
 352
 353         address &= PAGE_MASK;
 354         pgd = pgd_offset(mm, address);
 355         if (!pgd_present(*pgd))
 356                 goto kill;
 357
 358         pud = pud_offset(pgd, address);
 359         if (!pud_present(*pud))
 360                 goto kill;
 361
 362         pmd = pmd_offset(pud, address);
 363         if (!pmd_present(*pmd))
 364                 goto kill;
 365
 366         pte = pte_offset_kernel(pmd, address);
 367
 368         r = pte_read(*pte);
 369         w = pte_write(*pte);
 370         x = pte_exec(*pte);
 371         if (!pte_young(*pte)) {
 372                 r = 0;
 373                 w = 0;
 374         } else if (!pte_dirty(*pte)) {
 375                 w = 0;
 376         }
 377
 378         mm_id = &mm->context.skas.id;
 379         prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
 380                 (x ? UM_PROT_EXEC : 0));
 381         if (pte_newpage(*pte)) {
 382                 if (pte_present(*pte)) {
 383                         unsigned long long offset;
 384                         int fd;
 385
 386                         fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset);
 387                         err = map(mm_id, address, PAGE_SIZE, prot, fd, offset,
 388                                   1, &flush);
 389                 }
 390                 else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush);
 391         }
 392         else if (pte_newprot(*pte))
 393                 err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush);
 394
 395         if (err)
 396                 goto kill;
 397
 398         *pte = pte_mkuptodate(*pte);
 399
 400         return;
 401
 402 kill:
 403         printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address);
 404         force_sig(SIGKILL, current);
 405 }
 406
 407 pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
 408 {
 409         return pgd_offset(mm, address);
 410 }
 411
 412 pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
 413 {
 414         return pud_offset(pgd, address);
 415 }
 416
 417 pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
 418 {
 419         return pmd_offset(pud, address);
 420 }
 421
 422 pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
 423 {
 424         return pte_offset_kernel(pmd, address);
 425 }
 426
 427 pte_t *addr_pte(struct task_struct *task, unsigned long addr)
 428 {
 429         pgd_t *pgd = pgd_offset(task->mm, addr);
 430         pud_t *pud = pud_offset(pgd, addr);
 431         pmd_t *pmd = pmd_offset(pud, addr);
 432
 433         return pte_offset_map(pmd, addr);
 434 }
 435
 436 void flush_tlb_all(void)
 437 {
 438         flush_tlb_mm(current->mm);
 439 }
 440
 441 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 442 {
 443         flush_tlb_kernel_range_common(start, end);
 444 }
 445
 446 void flush_tlb_kernel_vm(void)
 447 {
 448         flush_tlb_kernel_range_common(start_vm, end_vm);
 449 }
 450
 451 void __flush_tlb_one(unsigned long addr)
 452 {
 453         flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
 454 }
 455
 456 static int do_ops(union mm_context *mmu, struct host_vm_op *ops, int last,
 457                   int finished, void **flush)
 458 {
 459         struct host_vm_op *op;
 460         int i, ret = 0;
 461
 462         for (i = 0; i <= last && !ret; i++) {
 463         op = &ops[i];
 464                 switch(op->type) {
 465                 case MMAP:
 466                         ret = map(&mmu->skas.id, op->u.mmap.addr,
 467                                   op->u.mmap.len, op->u.mmap.prot,
 468                                   op->u.mmap.fd, op->u.mmap.offset, finished,
 469                                   flush);
 470                         break;
 471                 case MUNMAP:
 472                         ret = unmap(&mmu->skas.id, op->u.munmap.addr,
 473                                     op->u.munmap.len, finished, flush);
 474                         break;
 475                 case MPROTECT:
 476                         ret = protect(&mmu->skas.id, op->u.mprotect.addr,
 477                                       op->u.mprotect.len, op->u.mprotect.prot,
 478                                       finished, flush);
 479                         break;
 480                 default:
 481                         printk(KERN_ERR "Unknown op type %d in do_ops\n",
 482                                op->type);
 483                         break;
 484                 }
 485         }
 486
 487         return ret;
 488 }
 489
 490 static void fix_range(struct mm_struct *mm, unsigned long start_addr,
 491                       unsigned long end_addr, int force)
 492 {
 493         if (!proc_mm && (end_addr > CONFIG_STUB_START))
 494                 end_addr = CONFIG_STUB_START;
 495
 496         fix_range_common(mm, start_addr, end_addr, force, do_ops);
 497 }
 498
 499 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 500                      unsigned long end)
 501 {
 502         if (vma->vm_mm == NULL)
 503                 flush_tlb_kernel_range_common(start, end);
 504         else fix_range(vma->vm_mm, start, end, 0);
 505 }
 506
 507 void flush_tlb_mm(struct mm_struct *mm)
 508 {
 509         unsigned long end;
 510
 511         /*
 512          * Don't bother flushing if this address space is about to be
 513          * destroyed.
 514          */
 515         if (atomic_read(&mm->mm_users) == 0)
 516                 return;
 517
 518         end = proc_mm ? task_size : CONFIG_STUB_START;
 519         fix_range(mm, 0, end, 0);
 520 }
 521
 522 void force_flush_all(void)
 523 {
 524         struct mm_struct *mm = current->mm;
 525         struct vm_area_struct *vma = mm->mmap;
 526
 527         while (vma != NULL) {
 528                 fix_range(mm, vma->vm_start, vma->vm_end, 1);
 529                 vma = vma->vm_next;
 530         }
 531 }