arch/alpha/kernel/smp.c

   1 /*
   2  *      linux/arch/alpha/kernel/smp.c
   3  *
   4  *      2001-07-09 Phil Ezolt (Phillip.Ezolt@compaq.com)
   5  *            Renamed modified smp_call_function to smp_call_function_on_cpu()
   6  *            Created an function that conforms to the old calling convention
   7  *            of smp_call_function().
   8  *
   9  *            This is helpful for DCPI.
  10  *
  11  */
  12
  13 #include <linux/errno.h>
  14 #include <linux/kernel.h>
  15 #include <linux/kernel_stat.h>
  16 #include <linux/module.h>
  17 #include <linux/sched.h>
  18 #include <linux/mm.h>
  19 #include <linux/err.h>
  20 #include <linux/threads.h>
  21 #include <linux/smp.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/init.h>
  24 #include <linux/delay.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/irq.h>
  27 #include <linux/cache.h>
  28 #include <linux/profile.h>
  29 #include <linux/bitops.h>
  30 #include <linux/cpu.h>
  31
  32 #include <asm/hwrpb.h>
  33 #include <asm/ptrace.h>
  34 #include <asm/atomic.h>
  35
  36 #include <asm/io.h>
  37 #include <asm/irq.h>
  38 #include <asm/pgtable.h>
  39 #include <asm/pgalloc.h>
  40 #include <asm/mmu_context.h>
  41 #include <asm/tlbflush.h>
  42
  43 #include "proto.h"
  44 #include "irq_impl.h"
  45
  46
  47 #define DEBUG_SMP 0
  48 #if DEBUG_SMP
  49 #define DBGS(args)      printk args
  50 #else
  51 #define DBGS(args)
  52 #endif
  53
  54 /* A collection of per-processor data.  */
  55 struct cpuinfo_alpha cpu_data[NR_CPUS];
  56 EXPORT_SYMBOL(cpu_data);
  57
  58 /* A collection of single bit ipi messages.  */
  59 static struct {
  60         unsigned long bits ____cacheline_aligned;
  61 } ipi_data[NR_CPUS] __cacheline_aligned;
  62
  63 enum ipi_message_type {
  64         IPI_RESCHEDULE,
  65         IPI_CALL_FUNC,
  66         IPI_CALL_FUNC_SINGLE,
  67         IPI_CPU_STOP,
  68 };
  69
  70 /* Set to a secondary's cpuid when it comes online.  */
  71 static int smp_secondary_alive __devinitdata = 0;
  72
  73 int smp_num_probed;             /* Internal processor count */
  74 int smp_num_cpus = 1;           /* Number that came online.  */
  75 EXPORT_SYMBOL(smp_num_cpus);
  76
  77 /*
  78  * Called by both boot and secondaries to move global data into
  79  *  per-processor storage.
  80  */
  81 static inline void __init
  82 smp_store_cpu_info(int cpuid)
  83 {
  84         cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
  85         cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
  86         cpu_data[cpuid].need_new_asn = 0;
  87         cpu_data[cpuid].asn_lock = 0;
  88 }
  89
  90 /*
  91  * Ideally sets up per-cpu profiling hooks.  Doesn't do much now...
  92  */
  93 static inline void __init
  94 smp_setup_percpu_timer(int cpuid)
  95 {
  96         cpu_data[cpuid].prof_counter = 1;
  97         cpu_data[cpuid].prof_multiplier = 1;
  98 }
  99
 100 static void __init
 101 wait_boot_cpu_to_stop(int cpuid)
 102 {
 103         unsigned long stop = jiffies + 10*HZ;
 104
 105         while (time_before(jiffies, stop)) {
 106                 if (!smp_secondary_alive)
 107                         return;
 108                 barrier();
 109         }
 110
 111         printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
 112         for (;;)
 113                 barrier();
 114 }
 115
 116 /*
 117  * Where secondaries begin a life of C.
 118  */
 119 void __cpuinit
 120 smp_callin(void)
 121 {
 122         int cpuid = hard_smp_processor_id();
 123
 124         if (cpu_online(cpuid)) {
 125                 printk("??, cpu 0x%x already present??\n", cpuid);
 126                 BUG();
 127         }
 128         set_cpu_online(cpuid, true);
 129
 130         /* Turn on machine checks.  */
 131         wrmces(7);
 132
 133         /* Set trap vectors.  */
 134         trap_init();
 135
 136         /* Set interrupt vector.  */
 137         wrent(entInt, 0);
 138
 139         /* Get our local ticker going. */
 140         smp_setup_percpu_timer(cpuid);
 141
 142         /* Call platform-specific callin, if specified */
 143         if (alpha_mv.smp_callin) alpha_mv.smp_callin();
 144
 145         /* All kernel threads share the same mm context.  */
 146         atomic_inc(&init_mm.mm_count);
 147         current->active_mm = &init_mm;
 148
 149         /* inform the notifiers about the new cpu */
 150         notify_cpu_starting(cpuid);
 151
 152         /* Must have completely accurate bogos.  */
 153         local_irq_enable();
 154
 155         /* Wait boot CPU to stop with irq enabled before running
 156            calibrate_delay. */
 157         wait_boot_cpu_to_stop(cpuid);
 158         mb();
 159         calibrate_delay();
 160
 161         smp_store_cpu_info(cpuid);
 162         /* Allow master to continue only after we written loops_per_jiffy.  */
 163         wmb();
 164         smp_secondary_alive = 1;
 165
 166         DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
 167               cpuid, current, current->active_mm));
 168
 169         /* Do nothing.  */
 170         cpu_idle();
 171 }
 172
 173 /* Wait until hwrpb->txrdy is clear for cpu.  Return -1 on timeout.  */
 174 static int __devinit
 175 wait_for_txrdy (unsigned long cpumask)
 176 {
 177         unsigned long timeout;
 178
 179         if (!(hwrpb->txrdy & cpumask))
 180                 return 0;
 181
 182         timeout = jiffies + 10*HZ;
 183         while (time_before(jiffies, timeout)) {
 184                 if (!(hwrpb->txrdy & cpumask))
 185                         return 0;
 186                 udelay(10);
 187                 barrier();
 188         }
 189
 190         return -1;
 191 }
 192
 193 /*
 194  * Send a message to a secondary's console.  "START" is one such
 195  * interesting message.  ;-)
 196  */
 197 static void __cpuinit
 198 send_secondary_console_msg(char *str, int cpuid)
 199 {
 200         struct percpu_struct *cpu;
 201         register char *cp1, *cp2;
 202         unsigned long cpumask;
 203         size_t len;
 204
 205         cpu = (struct percpu_struct *)
 206                 ((char*)hwrpb
 207                  + hwrpb->processor_offset
 208                  + cpuid * hwrpb->processor_size);
 209
 210         cpumask = (1UL << cpuid);
 211         if (wait_for_txrdy(cpumask))
 212                 goto timeout;
 213
 214         cp2 = str;
 215         len = strlen(cp2);
 216         *(unsigned int *)&cpu->ipc_buffer[0] = len;
 217         cp1 = (char *) &cpu->ipc_buffer[1];
 218         memcpy(cp1, cp2, len);
 219
 220         /* atomic test and set */
 221         wmb();
 222         set_bit(cpuid, &hwrpb->rxrdy);
 223
 224         if (wait_for_txrdy(cpumask))
 225                 goto timeout;
 226         return;
 227
 228  timeout:
 229         printk("Processor %x not ready\n", cpuid);
 230 }
 231
 232 /*
 233  * A secondary console wants to send a message.  Receive it.
 234  */
 235 static void
 236 recv_secondary_console_msg(void)
 237 {
 238         int mycpu, i, cnt;
 239         unsigned long txrdy = hwrpb->txrdy;
 240         char *cp1, *cp2, buf[80];
 241         struct percpu_struct *cpu;
 242
 243         DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));
 244
 245         mycpu = hard_smp_processor_id();
 246
 247         for (i = 0; i < NR_CPUS; i++) {
 248                 if (!(txrdy & (1UL << i)))
 249                         continue;
 250
 251                 DBGS(("recv_secondary_console_msg: "
 252                       "TXRDY contains CPU %d.\n", i));
 253
 254                 cpu = (struct percpu_struct *)
 255                   ((char*)hwrpb
 256                    + hwrpb->processor_offset
 257                    + i * hwrpb->processor_size);
 258
 259                 DBGS(("recv_secondary_console_msg: on %d from %d"
 260                       " HALT_REASON 0x%lx FLAGS 0x%lx\n",
 261                       mycpu, i, cpu->halt_reason, cpu->flags));
 262
 263                 cnt = cpu->ipc_buffer[0] >> 32;
 264                 if (cnt <= 0 || cnt >= 80)
 265                         strcpy(buf, "<<< BOGUS MSG >>>");
 266                 else {
 267                         cp1 = (char *) &cpu->ipc_buffer[11];
 268                         cp2 = buf;
 269                         strcpy(cp2, cp1);
 270
 271                         while ((cp2 = strchr(cp2, '\r')) != 0) {
 272                                 *cp2 = ' ';
 273                                 if (cp2[1] == '\n')
 274                                         cp2[1] = ' ';
 275                         }
 276                 }
 277
 278                 DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
 279                       "message is '%s'\n", mycpu, buf));
 280         }
 281
 282         hwrpb->txrdy = 0;
 283 }
 284
 285 /*
 286  * Convince the console to have a secondary cpu begin execution.
 287  */
 288 static int __cpuinit
 289 secondary_cpu_start(int cpuid, struct task_struct *idle)
 290 {
 291         struct percpu_struct *cpu;
 292         struct pcb_struct *hwpcb, *ipcb;
 293         unsigned long timeout;
 294
 295         cpu = (struct percpu_struct *)
 296                 ((char*)hwrpb
 297                  + hwrpb->processor_offset
 298                  + cpuid * hwrpb->processor_size);
 299         hwpcb = (struct pcb_struct *) cpu->hwpcb;
 300         ipcb = &task_thread_info(idle)->pcb;
 301
 302         /* Initialize the CPU's HWPCB to something just good enough for
 303            us to get started.  Immediately after starting, we'll swpctx
 304            to the target idle task's pcb.  Reuse the stack in the mean
 305            time.  Precalculate the target PCBB.  */
 306         hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
 307         hwpcb->usp = 0;
 308         hwpcb->ptbr = ipcb->ptbr;
 309         hwpcb->pcc = 0;
 310         hwpcb->asn = 0;
 311         hwpcb->unique = virt_to_phys(ipcb);
 312         hwpcb->flags = ipcb->flags;
 313         hwpcb->res1 = hwpcb->res2 = 0;
 314
 315 #if 0
 316         DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
 317               hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
 318 #endif
 319         DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
 320               cpuid, idle->state, ipcb->flags));
 321
 322         /* Setup HWRPB fields that SRM uses to activate secondary CPU */
 323         hwrpb->CPU_restart = __smp_callin;
 324         hwrpb->CPU_restart_data = (unsigned long) __smp_callin;
 325
 326         /* Recalculate and update the HWRPB checksum */
 327         hwrpb_update_checksum(hwrpb);
 328
 329         /*
 330          * Send a "start" command to the specified processor.
 331          */
 332
 333         /* SRM III 3.4.1.3 */
 334         cpu->flags |= 0x22;     /* turn on Context Valid and Restart Capable */
 335         cpu->flags &= ~1;       /* turn off Bootstrap In Progress */
 336         wmb();
 337
 338         send_secondary_console_msg("START\r\n", cpuid);
 339
 340         /* Wait 10 seconds for an ACK from the console.  */
 341         timeout = jiffies + 10*HZ;
 342         while (time_before(jiffies, timeout)) {
 343                 if (cpu->flags & 1)
 344                         goto started;
 345                 udelay(10);
 346                 barrier();
 347         }
 348         printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
 349         return -1;
 350
 351  started:
 352         DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
 353         return 0;
 354 }
 355
 356 /*
 357  * Bring one cpu online.
 358  */
 359 static int __cpuinit
 360 smp_boot_one_cpu(int cpuid)
 361 {
 362         struct task_struct *idle;
 363         unsigned long timeout;
 364
 365         /* Cook up an idler for this guy.  Note that the address we
 366            give to kernel_thread is irrelevant -- it's going to start
 367            where HWRPB.CPU_restart says to start.  But this gets all
 368            the other task-y sort of data structures set up like we
 369            wish.  We can't use kernel_thread since we must avoid
 370            rescheduling the child.  */
 371         idle = fork_idle(cpuid);
 372         if (IS_ERR(idle))
 373                 panic("failed fork for CPU %d", cpuid);
 374
 375         DBGS(("smp_boot_one_cpu: CPU %d state 0x%lx flags 0x%lx\n",
 376               cpuid, idle->state, idle->flags));
 377
 378         /* Signal the secondary to wait a moment.  */
 379         smp_secondary_alive = -1;
 380
 381         /* Whirrr, whirrr, whirrrrrrrrr... */
 382         if (secondary_cpu_start(cpuid, idle))
 383                 return -1;
 384
 385         /* Notify the secondary CPU it can run calibrate_delay.  */
 386         mb();
 387         smp_secondary_alive = 0;
 388
 389         /* We've been acked by the console; wait one second for
 390            the task to start up for real.  */
 391         timeout = jiffies + 1*HZ;
 392         while (time_before(jiffies, timeout)) {
 393                 if (smp_secondary_alive == 1)
 394                         goto alive;
 395                 udelay(10);
 396                 barrier();
 397         }
 398
 399         /* We failed to boot the CPU.  */
 400
 401         printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
 402         return -1;
 403
 404  alive:
 405         /* Another "Red Snapper". */
 406         return 0;
 407 }
 408
 409 /*
 410  * Called from setup_arch.  Detect an SMP system and which processors
 411  * are present.
 412  */
 413 void __init
 414 setup_smp(void)
 415 {
 416         struct percpu_struct *cpubase, *cpu;
 417         unsigned long i;
 418
 419         if (boot_cpuid != 0) {
 420                 printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
 421                        boot_cpuid);
 422         }
 423
 424         if (hwrpb->nr_processors > 1) {
 425                 int boot_cpu_palrev;
 426
 427                 DBGS(("setup_smp: nr_processors %ld\n",
 428                       hwrpb->nr_processors));
 429
 430                 cpubase = (struct percpu_struct *)
 431                         ((char*)hwrpb + hwrpb->processor_offset);
 432                 boot_cpu_palrev = cpubase->pal_revision;
 433
 434                 for (i = 0; i < hwrpb->nr_processors; i++) {
 435                         cpu = (struct percpu_struct *)
 436                                 ((char *)cpubase + i*hwrpb->processor_size);
 437                         if ((cpu->flags & 0x1cc) == 0x1cc) {
 438                                 smp_num_probed++;
 439                                 set_cpu_possible(i, true);
 440                                 set_cpu_present(i, true);
 441                                 cpu->pal_revision = boot_cpu_palrev;
 442                         }
 443
 444                         DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
 445                               i, cpu->flags, cpu->type));
 446                         DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
 447                               i, cpu->pal_revision));
 448                 }
 449         } else {
 450                 smp_num_probed = 1;
 451         }
 452
 453         printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_map = %lx\n",
 454                smp_num_probed, cpu_present_map.bits[0]);
 455 }
 456
 457 /*
 458  * Called by smp_init prepare the secondaries
 459  */
 460 void __init
 461 smp_prepare_cpus(unsigned int max_cpus)
 462 {
 463         /* Take care of some initial bookkeeping.  */
 464         memset(ipi_data, 0, sizeof(ipi_data));
 465
 466         current_thread_info()->cpu = boot_cpuid;
 467
 468         smp_store_cpu_info(boot_cpuid);
 469         smp_setup_percpu_timer(boot_cpuid);
 470
 471         /* Nothing to do on a UP box, or when told not to.  */
 472         if (smp_num_probed == 1 || max_cpus == 0) {
 473                 init_cpu_possible(cpumask_of(boot_cpuid));
 474                 init_cpu_present(cpumask_of(boot_cpuid));
 475                 printk(KERN_INFO "SMP mode deactivated.\n");
 476                 return;
 477         }
 478
 479         printk(KERN_INFO "SMP starting up secondaries.\n");
 480
 481         smp_num_cpus = smp_num_probed;
 482 }
 483
 484 void __devinit
 485 smp_prepare_boot_cpu(void)
 486 {
 487 }
 488
 489 int __cpuinit
 490 __cpu_up(unsigned int cpu)
 491 {
 492         smp_boot_one_cpu(cpu);
 493
 494         return cpu_online(cpu) ? 0 : -ENOSYS;
 495 }
 496
 497 void __init
 498 smp_cpus_done(unsigned int max_cpus)
 499 {
 500         int cpu;
 501         unsigned long bogosum = 0;
 502
 503         for(cpu = 0; cpu < NR_CPUS; cpu++)
 504                 if (cpu_online(cpu))
 505                         bogosum += cpu_data[cpu].loops_per_jiffy;
 506
 507         printk(KERN_INFO "SMP: Total of %d processors activated "
 508                "(%lu.%02lu BogoMIPS).\n",
 509                num_online_cpus(),
 510                (bogosum + 2500) / (500000/HZ),
 511                ((bogosum + 2500) / (5000/HZ)) % 100);
 512 }
 513
 514 \f
 515 void
 516 smp_percpu_timer_interrupt(struct pt_regs *regs)
 517 {
 518         struct pt_regs *old_regs;
 519         int cpu = smp_processor_id();
 520         unsigned long user = user_mode(regs);
 521         struct cpuinfo_alpha *data = &cpu_data[cpu];
 522
 523         old_regs = set_irq_regs(regs);
 524
 525         /* Record kernel PC.  */
 526         profile_tick(CPU_PROFILING);
 527
 528         if (!--data->prof_counter) {
 529                 /* We need to make like a normal interrupt -- otherwise
 530                    timer interrupts ignore the global interrupt lock,
 531                    which would be a Bad Thing.  */
 532                 irq_enter();
 533
 534                 update_process_times(user);
 535
 536                 data->prof_counter = data->prof_multiplier;
 537
 538                 irq_exit();
 539         }
 540         set_irq_regs(old_regs);
 541 }
 542
 543 int
 544 setup_profiling_timer(unsigned int multiplier)
 545 {
 546         return -EINVAL;
 547 }
 548
 549 \f
 550 static void
 551 send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
 552 {
 553         int i;
 554
 555         mb();
 556         for_each_cpu(i, to_whom)
 557                 set_bit(operation, &ipi_data[i].bits);
 558
 559         mb();
 560         for_each_cpu(i, to_whom)
 561                 wripir(i);
 562 }
 563
 564 void
 565 handle_ipi(struct pt_regs *regs)
 566 {
 567         int this_cpu = smp_processor_id();
 568         unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
 569         unsigned long ops;
 570
 571 #if 0
 572         DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
 573               this_cpu, *pending_ipis, regs->pc));
 574 #endif
 575
 576         mb();   /* Order interrupt and bit testing. */
 577         while ((ops = xchg(pending_ipis, 0)) != 0) {
 578           mb(); /* Order bit clearing and data access. */
 579           do {
 580                 unsigned long which;
 581
 582                 which = ops & -ops;
 583                 ops &= ~which;
 584                 which = __ffs(which);
 585
 586                 switch (which) {
 587                 case IPI_RESCHEDULE:
 588                         scheduler_ipi();
 589                         break;
 590
 591                 case IPI_CALL_FUNC:
 592                         generic_smp_call_function_interrupt();
 593                         break;
 594
 595                 case IPI_CALL_FUNC_SINGLE:
 596                         generic_smp_call_function_single_interrupt();
 597                         break;
 598
 599                 case IPI_CPU_STOP:
 600                         halt();
 601
 602                 default:
 603                         printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
 604                                this_cpu, which);
 605                         break;
 606                 }
 607           } while (ops);
 608
 609           mb(); /* Order data access and bit testing. */
 610         }
 611
 612         cpu_data[this_cpu].ipi_count++;
 613
 614         if (hwrpb->txrdy)
 615                 recv_secondary_console_msg();
 616 }
 617
 618 void
 619 smp_send_reschedule(int cpu)
 620 {
 621 #ifdef DEBUG_IPI_MSG
 622         if (cpu == hard_smp_processor_id())
 623                 printk(KERN_WARNING
 624                        "smp_send_reschedule: Sending IPI to self.\n");
 625 #endif
 626         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
 627 }
 628
 629 void
 630 smp_send_stop(void)
 631 {
 632         cpumask_t to_whom = cpu_possible_map;
 633         cpu_clear(smp_processor_id(), to_whom);
 634 #ifdef DEBUG_IPI_MSG
 635         if (hard_smp_processor_id() != boot_cpu_id)
 636                 printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
 637 #endif
 638         send_ipi_message(&to_whom, IPI_CPU_STOP);
 639 }
 640
 641 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 642 {
 643         send_ipi_message(mask, IPI_CALL_FUNC);
 644 }
 645
 646 void arch_send_call_function_single_ipi(int cpu)
 647 {
 648         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
 649 }
 650
 651 static void
 652 ipi_imb(void *ignored)
 653 {
 654         imb();
 655 }
 656
 657 void
 658 smp_imb(void)
 659 {
 660         /* Must wait other processors to flush their icache before continue. */
 661         if (on_each_cpu(ipi_imb, NULL, 1))
 662                 printk(KERN_CRIT "smp_imb: timed out\n");
 663 }
 664 EXPORT_SYMBOL(smp_imb);
 665
 666 static void
 667 ipi_flush_tlb_all(void *ignored)
 668 {
 669         tbia();
 670 }
 671
 672 void
 673 flush_tlb_all(void)
 674 {
 675         /* Although we don't have any data to pass, we do want to
 676            synchronize with the other processors.  */
 677         if (on_each_cpu(ipi_flush_tlb_all, NULL, 1)) {
 678                 printk(KERN_CRIT "flush_tlb_all: timed out\n");
 679         }
 680 }
 681
 682 #define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
 683
 684 static void
 685 ipi_flush_tlb_mm(void *x)
 686 {
 687         struct mm_struct *mm = (struct mm_struct *) x;
 688         if (mm == current->active_mm && !asn_locked())
 689                 flush_tlb_current(mm);
 690         else
 691                 flush_tlb_other(mm);
 692 }
 693
 694 void
 695 flush_tlb_mm(struct mm_struct *mm)
 696 {
 697         preempt_disable();
 698
 699         if (mm == current->active_mm) {
 700                 flush_tlb_current(mm);
 701                 if (atomic_read(&mm->mm_users) <= 1) {
 702                         int cpu, this_cpu = smp_processor_id();
 703                         for (cpu = 0; cpu < NR_CPUS; cpu++) {
 704                                 if (!cpu_online(cpu) || cpu == this_cpu)
 705                                         continue;
 706                                 if (mm->context[cpu])
 707                                         mm->context[cpu] = 0;
 708                         }
 709                         preempt_enable();
 710                         return;
 711                 }
 712         }
 713
 714         if (smp_call_function(ipi_flush_tlb_mm, mm, 1)) {
 715                 printk(KERN_CRIT "flush_tlb_mm: timed out\n");
 716         }
 717
 718         preempt_enable();
 719 }
 720 EXPORT_SYMBOL(flush_tlb_mm);
 721
 722 struct flush_tlb_page_struct {
 723         struct vm_area_struct *vma;
 724         struct mm_struct *mm;
 725         unsigned long addr;
 726 };
 727
 728 static void
 729 ipi_flush_tlb_page(void *x)
 730 {
 731         struct flush_tlb_page_struct *data = (struct flush_tlb_page_struct *)x;
 732         struct mm_struct * mm = data->mm;
 733
 734         if (mm == current->active_mm && !asn_locked())
 735                 flush_tlb_current_page(mm, data->vma, data->addr);
 736         else
 737                 flush_tlb_other(mm);
 738 }
 739
 740 void
 741 flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 742 {
 743         struct flush_tlb_page_struct data;
 744         struct mm_struct *mm = vma->vm_mm;
 745
 746         preempt_disable();
 747
 748         if (mm == current->active_mm) {
 749                 flush_tlb_current_page(mm, vma, addr);
 750                 if (atomic_read(&mm->mm_users) <= 1) {
 751                         int cpu, this_cpu = smp_processor_id();
 752                         for (cpu = 0; cpu < NR_CPUS; cpu++) {
 753                                 if (!cpu_online(cpu) || cpu == this_cpu)
 754                                         continue;
 755                                 if (mm->context[cpu])
 756                                         mm->context[cpu] = 0;
 757                         }
 758                         preempt_enable();
 759                         return;
 760                 }
 761         }
 762
 763         data.vma = vma;
 764         data.mm = mm;
 765         data.addr = addr;
 766
 767         if (smp_call_function(ipi_flush_tlb_page, &data, 1)) {
 768                 printk(KERN_CRIT "flush_tlb_page: timed out\n");
 769         }
 770
 771         preempt_enable();
 772 }
 773 EXPORT_SYMBOL(flush_tlb_page);
 774
 775 void
 776 flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 777 {
 778         /* On the Alpha we always flush the whole user tlb.  */
 779         flush_tlb_mm(vma->vm_mm);
 780 }
 781 EXPORT_SYMBOL(flush_tlb_range);
 782
 783 static void
 784 ipi_flush_icache_page(void *x)
 785 {
 786         struct mm_struct *mm = (struct mm_struct *) x;
 787         if (mm == current->active_mm && !asn_locked())
 788                 __load_new_mm_context(mm);
 789         else
 790                 flush_tlb_other(mm);
 791 }
 792
 793 void
 794 flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
 795                         unsigned long addr, int len)
 796 {
 797         struct mm_struct *mm = vma->vm_mm;
 798
 799         if ((vma->vm_flags & VM_EXEC) == 0)
 800                 return;
 801
 802         preempt_disable();
 803
 804         if (mm == current->active_mm) {
 805                 __load_new_mm_context(mm);
 806                 if (atomic_read(&mm->mm_users) <= 1) {
 807                         int cpu, this_cpu = smp_processor_id();
 808                         for (cpu = 0; cpu < NR_CPUS; cpu++) {
 809                                 if (!cpu_online(cpu) || cpu == this_cpu)
 810                                         continue;
 811                                 if (mm->context[cpu])
 812                                         mm->context[cpu] = 0;
 813                         }
 814                         preempt_enable();
 815                         return;
 816                 }
 817         }
 818
 819         if (smp_call_function(ipi_flush_icache_page, mm, 1)) {
 820                 printk(KERN_CRIT "flush_icache_page: timed out\n");
 821         }
 822
 823         preempt_enable();
 824 }