arch/um/kernel/tlb.c

   1 /*
   2  * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com)
   3  * Licensed under the GPL
   4  */
   5
   6 #include "linux/mm.h"
   7 #include "asm/page.h"
   8 #include "asm/pgalloc.h"
   9 #include "asm/tlbflush.h"
  10 #include "choose-mode.h"
  11 #include "mode_kern.h"
  12 #include "user_util.h"
  13 #include "tlb.h"
  14 #include "mem.h"
  15 #include "mem_user.h"
  16 #include "os.h"
  17
  18 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
  19
  20 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
  21                       unsigned long end_addr, int force,
  22                       void (*do_ops)(union mm_context *, struct host_vm_op *,
  23                                      int))
  24 {
  25         pgd_t *npgd;
  26         pud_t *npud;
  27         pmd_t *npmd;
  28         pte_t *npte;
  29         union mm_context *mmu = &mm->context;
  30         unsigned long addr, end;
  31         int r, w, x;
  32         struct host_vm_op ops[16];
  33         int op_index = -1, last_op = sizeof(ops) / sizeof(ops[0]) - 1;
  34
  35         if(mm == NULL) return;
  36
  37         for(addr = start_addr; addr < end_addr;){
  38                 npgd = pgd_offset(mm, addr);
  39                 if(!pgd_present(*npgd)){
  40                         end = ADD_ROUND(addr, PGDIR_SIZE);
  41                         if(end > end_addr)
  42                                 end = end_addr;
  43                         if(force || pgd_newpage(*npgd)){
  44                                 op_index = add_munmap(addr, end - addr, ops,
  45                                                       op_index, last_op, mmu,
  46                                                       do_ops);
  47                                 pgd_mkuptodate(*npgd);
  48                         }
  49                         addr = end;
  50                         continue;
  51                 }
  52
  53                 npud = pud_offset(npgd, addr);
  54                 if(!pud_present(*npud)){
  55                         end = ADD_ROUND(addr, PUD_SIZE);
  56                         if(end > end_addr)
  57                                 end = end_addr;
  58                         if(force || pud_newpage(*npud)){
  59                                 op_index = add_munmap(addr, end - addr, ops,
  60                                                       op_index, last_op, mmu,
  61                                                       do_ops);
  62                                 pud_mkuptodate(*npud);
  63                         }
  64                         addr = end;
  65                         continue;
  66                 }
  67
  68                 npmd = pmd_offset(npud, addr);
  69                 if(!pmd_present(*npmd)){
  70                         end = ADD_ROUND(addr, PMD_SIZE);
  71                         if(end > end_addr)
  72                                 end = end_addr;
  73                         if(force || pmd_newpage(*npmd)){
  74                                 op_index = add_munmap(addr, end - addr, ops,
  75                                                       op_index, last_op, mmu,
  76                                                       do_ops);
  77                                 pmd_mkuptodate(*npmd);
  78                         }
  79                         addr = end;
  80                         continue;
  81                 }
  82
  83                 npte = pte_offset_kernel(npmd, addr);
  84                 r = pte_read(*npte);
  85                 w = pte_write(*npte);
  86                 x = pte_exec(*npte);
  87                 if(!pte_dirty(*npte))
  88                         w = 0;
  89                 if(!pte_young(*npte)){
  90                         r = 0;
  91                         w = 0;
  92                 }
  93                 if(force || pte_newpage(*npte)){
  94                         if(pte_present(*npte))
  95                                 op_index = add_mmap(addr,
  96                                                     pte_val(*npte) & PAGE_MASK,
  97                                                     PAGE_SIZE, r, w, x, ops,
  98                                                     op_index, last_op, mmu,
  99                                                     do_ops);
 100                         else op_index = add_munmap(addr, PAGE_SIZE, ops,
 101                                                    op_index, last_op, mmu,
 102                                                    do_ops);
 103                 }
 104                 else if(pte_newprot(*npte))
 105                         op_index = add_mprotect(addr, PAGE_SIZE, r, w, x, ops,
 106                                                 op_index, last_op, mmu,
 107                                                 do_ops);
 108
 109                 *npte = pte_mkuptodate(*npte);
 110                 addr += PAGE_SIZE;
 111         }
 112         (*do_ops)(mmu, ops, op_index);
 113 }
 114
 115 int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
 116 {
 117         struct mm_struct *mm;
 118         pgd_t *pgd;
 119         pud_t *pud;
 120         pmd_t *pmd;
 121         pte_t *pte;
 122         unsigned long addr, last;
 123         int updated = 0, err;
 124
 125         mm = &init_mm;
 126         for(addr = start; addr < end;){
 127                 pgd = pgd_offset(mm, addr);
 128                 if(!pgd_present(*pgd)){
 129                         last = ADD_ROUND(addr, PGDIR_SIZE);
 130                         if(last > end)
 131                                 last = end;
 132                         if(pgd_newpage(*pgd)){
 133                                 updated = 1;
 134                                 err = os_unmap_memory((void *) addr,
 135                                                       last - addr);
 136                                 if(err < 0)
 137                                         panic("munmap failed, errno = %d\n",
 138                                               -err);
 139                         }
 140                         addr = last;
 141                         continue;
 142                 }
 143
 144                 pud = pud_offset(pgd, addr);
 145                 if(!pud_present(*pud)){
 146                         last = ADD_ROUND(addr, PUD_SIZE);
 147                         if(last > end)
 148                                 last = end;
 149                         if(pud_newpage(*pud)){
 150                                 updated = 1;
 151                                 err = os_unmap_memory((void *) addr,
 152                                                       last - addr);
 153                                 if(err < 0)
 154                                         panic("munmap failed, errno = %d\n",
 155                                               -err);
 156                         }
 157                         addr = last;
 158                         continue;
 159                 }
 160
 161                 pmd = pmd_offset(pud, addr);
 162                 if(!pmd_present(*pmd)){
 163                         last = ADD_ROUND(addr, PMD_SIZE);
 164                         if(last > end)
 165                                 last = end;
 166                         if(pmd_newpage(*pmd)){
 167                                 updated = 1;
 168                                 err = os_unmap_memory((void *) addr,
 169                                                       last - addr);
 170                                 if(err < 0)
 171                                         panic("munmap failed, errno = %d\n",
 172                                               -err);
 173                         }
 174                         addr = last;
 175                         continue;
 176                 }
 177
 178                 pte = pte_offset_kernel(pmd, addr);
 179                 if(!pte_present(*pte) || pte_newpage(*pte)){
 180                         updated = 1;
 181                         err = os_unmap_memory((void *) addr,
 182                                               PAGE_SIZE);
 183                         if(err < 0)
 184                                 panic("munmap failed, errno = %d\n",
 185                                       -err);
 186                         if(pte_present(*pte))
 187                                 map_memory(addr,
 188                                            pte_val(*pte) & PAGE_MASK,
 189                                            PAGE_SIZE, 1, 1, 1);
 190                 }
 191                 else if(pte_newprot(*pte)){
 192                         updated = 1;
 193                         protect_memory(addr, PAGE_SIZE, 1, 1, 1, 1);
 194                 }
 195                 addr += PAGE_SIZE;
 196         }
 197         return(updated);
 198 }
 199
 200 pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
 201 {
 202         return(pgd_offset(mm, address));
 203 }
 204
 205 pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
 206 {
 207         return(pud_offset(pgd, address));
 208 }
 209
 210 pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
 211 {
 212         return(pmd_offset(pud, address));
 213 }
 214
 215 pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
 216 {
 217         return(pte_offset_kernel(pmd, address));
 218 }
 219
 220 pte_t *addr_pte(struct task_struct *task, unsigned long addr)
 221 {
 222         pgd_t *pgd = pgd_offset(task->mm, addr);
 223         pud_t *pud = pud_offset(pgd, addr);
 224         pmd_t *pmd = pmd_offset(pud, addr);
 225
 226         return(pte_offset_map(pmd, addr));
 227 }
 228
 229 int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
 230              int r, int w, int x, struct host_vm_op *ops, int index,
 231              int last_filled, union mm_context *mmu,
 232              void (*do_ops)(union mm_context *, struct host_vm_op *, int))
 233 {
 234         __u64 offset;
 235         struct host_vm_op *last;
 236         int fd;
 237
 238         fd = phys_mapping(phys, &offset);
 239         if(index != -1){
 240                 last = &ops[index];
 241                 if((last->type == MMAP) &&
 242                    (last->u.mmap.addr + last->u.mmap.len == virt) &&
 243                    (last->u.mmap.r == r) && (last->u.mmap.w == w) &&
 244                    (last->u.mmap.x == x) && (last->u.mmap.fd == fd) &&
 245                    (last->u.mmap.offset + last->u.mmap.len == offset)){
 246                         last->u.mmap.len += len;
 247                         return(index);
 248                 }
 249         }
 250
 251         if(index == last_filled){
 252                 (*do_ops)(mmu, ops, last_filled);
 253                 index = -1;
 254         }
 255
 256         ops[++index] = ((struct host_vm_op) { .type     = MMAP,
 257                                               .u = { .mmap = {
 258                                                       .addr     = virt,
 259                                                       .len      = len,
 260                                                       .r        = r,
 261                                                       .w        = w,
 262                                                       .x        = x,
 263                                                       .fd       = fd,
 264                                                       .offset   = offset }
 265                                               } });
 266         return(index);
 267 }
 268
 269 int add_munmap(unsigned long addr, unsigned long len, struct host_vm_op *ops,
 270                int index, int last_filled, union mm_context *mmu,
 271                void (*do_ops)(union mm_context *, struct host_vm_op *, int))
 272 {
 273         struct host_vm_op *last;
 274
 275         if(index != -1){
 276                 last = &ops[index];
 277                 if((last->type == MUNMAP) &&
 278                    (last->u.munmap.addr + last->u.mmap.len == addr)){
 279                         last->u.munmap.len += len;
 280                         return(index);
 281                 }
 282         }
 283
 284         if(index == last_filled){
 285                 (*do_ops)(mmu, ops, last_filled);
 286                 index = -1;
 287         }
 288
 289         ops[++index] = ((struct host_vm_op) { .type     = MUNMAP,
 290                                               .u = { .munmap = {
 291                                                       .addr     = addr,
 292                                                       .len      = len } } });
 293         return(index);
 294 }
 295
 296 int add_mprotect(unsigned long addr, unsigned long len, int r, int w, int x,
 297                  struct host_vm_op *ops, int index, int last_filled,
 298                  union mm_context *mmu,
 299                  void (*do_ops)(union mm_context *, struct host_vm_op *, int))
 300 {
 301         struct host_vm_op *last;
 302
 303         if(index != -1){
 304                 last = &ops[index];
 305                 if((last->type == MPROTECT) &&
 306                    (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
 307                    (last->u.mprotect.r == r) && (last->u.mprotect.w == w) &&
 308                    (last->u.mprotect.x == x)){
 309                         last->u.mprotect.len += len;
 310                         return(index);
 311                 }
 312         }
 313
 314         if(index == last_filled){
 315                 (*do_ops)(mmu, ops, last_filled);
 316                 index = -1;
 317         }
 318
 319         ops[++index] = ((struct host_vm_op) { .type     = MPROTECT,
 320                                               .u = { .mprotect = {
 321                                                       .addr     = addr,
 322                                                       .len      = len,
 323                                                       .r        = r,
 324                                                       .w        = w,
 325                                                       .x        = x } } });
 326         return(index);
 327 }
 328
 329 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
 330 {
 331         address &= PAGE_MASK;
 332         flush_tlb_range(vma, address, address + PAGE_SIZE);
 333 }
 334
 335 void flush_tlb_all(void)
 336 {
 337         flush_tlb_mm(current->mm);
 338 }
 339
 340 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 341 {
 342         CHOOSE_MODE_PROC(flush_tlb_kernel_range_tt,
 343                          flush_tlb_kernel_range_common, start, end);
 344 }
 345
 346 void flush_tlb_kernel_vm(void)
 347 {
 348         CHOOSE_MODE(flush_tlb_kernel_vm_tt(),
 349                     flush_tlb_kernel_range_common(start_vm, end_vm));
 350 }
 351
 352 void __flush_tlb_one(unsigned long addr)
 353 {
 354         CHOOSE_MODE_PROC(__flush_tlb_one_tt, __flush_tlb_one_skas, addr);
 355 }
 356
 357 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 358                      unsigned long end)
 359 {
 360         CHOOSE_MODE_PROC(flush_tlb_range_tt, flush_tlb_range_skas, vma, start,
 361                          end);
 362 }
 363
 364 void flush_tlb_mm(struct mm_struct *mm)
 365 {
 366         CHOOSE_MODE_PROC(flush_tlb_mm_tt, flush_tlb_mm_skas, mm);
 367 }
 368
 369 void force_flush_all(void)
 370 {
 371         CHOOSE_MODE(force_flush_all_tt(), force_flush_all_skas());
 372 }
 373