arch/x86/kernel/traps_32.c

   1 /*
   2  *  Copyright (C) 1991, 1992  Linus Torvalds
   3  *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
   4  *
   5  *  Pentium III FXSR, SSE support
   6  *      Gareth Hughes <gareth@valinux.com>, May 2000
   7  */
   8
   9 /*
  10  * 'Traps.c' handles hardware traps and faults after we have saved some
  11  * state in 'asm.s'.
  12  */
  13 #include <linux/interrupt.h>
  14 #include <linux/kallsyms.h>
  15 #include <linux/spinlock.h>
  16 #include <linux/highmem.h>
  17 #include <linux/kprobes.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/utsname.h>
  20 #include <linux/kdebug.h>
  21 #include <linux/kernel.h>
  22 #include <linux/module.h>
  23 #include <linux/ptrace.h>
  24 #include <linux/string.h>
  25 #include <linux/unwind.h>
  26 #include <linux/delay.h>
  27 #include <linux/errno.h>
  28 #include <linux/kexec.h>
  29 #include <linux/sched.h>
  30 #include <linux/timer.h>
  31 #include <linux/init.h>
  32 #include <linux/bug.h>
  33 #include <linux/nmi.h>
  34 #include <linux/mm.h>
  35
  36 #ifdef CONFIG_EISA
  37 #include <linux/ioport.h>
  38 #include <linux/eisa.h>
  39 #endif
  40
  41 #ifdef CONFIG_MCA
  42 #include <linux/mca.h>
  43 #endif
  44
  45 #if defined(CONFIG_EDAC)
  46 #include <linux/edac.h>
  47 #endif
  48
  49 #include <asm/processor-flags.h>
  50 #include <asm/arch_hooks.h>
  51 #include <asm/stacktrace.h>
  52 #include <asm/processor.h>
  53 #include <asm/debugreg.h>
  54 #include <asm/atomic.h>
  55 #include <asm/system.h>
  56 #include <asm/unwind.h>
  57 #include <asm/desc.h>
  58 #include <asm/i387.h>
  59 #include <asm/nmi.h>
  60 #include <asm/smp.h>
  61 #include <asm/io.h>
  62 #include <asm/traps.h>
  63
  64 #include "mach_traps.h"
  65
  66 DECLARE_BITMAP(used_vectors, NR_VECTORS);
  67 EXPORT_SYMBOL_GPL(used_vectors);
  68
  69 asmlinkage int system_call(void);
  70
  71 /* Do we ignore FPU interrupts ? */
  72 char ignore_fpu_irq;
  73
  74 /*
  75  * The IDT has to be page-aligned to simplify the Pentium
  76  * F0 0F bug workaround.. We have a special link segment
  77  * for this.
  78  */
  79 gate_desc idt_table[256]
  80         __attribute__((__section__(".data.idt"))) = { { { { 0, 0 } } }, };
  81
  82 int panic_on_unrecovered_nmi;
  83 int kstack_depth_to_print = 24;
  84 static unsigned int code_bytes = 64;
  85 static int ignore_nmis;
  86 static int die_counter;
  87
  88 static inline void conditional_sti(struct pt_regs *regs)
  89 {
  90         if (regs->flags & X86_EFLAGS_IF)
  91                 local_irq_enable();
  92 }
  93
  94 void printk_address(unsigned long address, int reliable)
  95 {
  96 #ifdef CONFIG_KALLSYMS
  97         unsigned long offset = 0;
  98         unsigned long symsize;
  99         const char *symname;
 100         char *modname;
 101         char *delim = ":";
 102         char namebuf[KSYM_NAME_LEN];
 103         char reliab[4] = "";
 104
 105         symname = kallsyms_lookup(address, &symsize, &offset,
 106                                         &modname, namebuf);
 107         if (!symname) {
 108                 printk(" [<%08lx>]\n", address);
 109                 return;
 110         }
 111         if (!reliable)
 112                 strcpy(reliab, "? ");
 113
 114         if (!modname)
 115                 modname = delim = "";
 116         printk(" [<%08lx>] %s%s%s%s%s+0x%lx/0x%lx\n",
 117                 address, reliab, delim, modname, delim, symname, offset, symsize);
 118 #else
 119         printk(" [<%08lx>]\n", address);
 120 #endif
 121 }
 122
 123 static inline int valid_stack_ptr(struct thread_info *tinfo,
 124                         void *p, unsigned int size)
 125 {
 126         void *t = tinfo;
 127         return  p > t && p <= t + THREAD_SIZE - size;
 128 }
 129
 130 /* The form of the top of the frame on the stack */
 131 struct stack_frame {
 132         struct stack_frame *next_frame;
 133         unsigned long return_address;
 134 };
 135
 136 static inline unsigned long
 137 print_context_stack(struct thread_info *tinfo,
 138                 unsigned long *stack, unsigned long bp,
 139                 const struct stacktrace_ops *ops, void *data)
 140 {
 141         struct stack_frame *frame = (struct stack_frame *)bp;
 142
 143         while (valid_stack_ptr(tinfo, stack, sizeof(*stack))) {
 144                 unsigned long addr;
 145
 146                 addr = *stack;
 147                 if (__kernel_text_address(addr)) {
 148                         if ((unsigned long) stack == bp + 4) {
 149                                 ops->address(data, addr, 1);
 150                                 frame = frame->next_frame;
 151                                 bp = (unsigned long) frame;
 152                         } else {
 153                                 ops->address(data, addr, bp == 0);
 154                         }
 155                 }
 156                 stack++;
 157         }
 158         return bp;
 159 }
 160
 161 void dump_trace(struct task_struct *task, struct pt_regs *regs,
 162                 unsigned long *stack, unsigned long bp,
 163                 const struct stacktrace_ops *ops, void *data)
 164 {
 165         if (!task)
 166                 task = current;
 167
 168         if (!stack) {
 169                 unsigned long dummy;
 170                 stack = &dummy;
 171                 if (task != current)
 172                         stack = (unsigned long *)task->thread.sp;
 173         }
 174
 175 #ifdef CONFIG_FRAME_POINTER
 176         if (!bp) {
 177                 if (task == current) {
 178                         /* Grab bp right from our regs */
 179                         asm("movl %%ebp, %0" : "=r" (bp) :);
 180                 } else {
 181                         /* bp is the last reg pushed by switch_to */
 182                         bp = *(unsigned long *) task->thread.sp;
 183                 }
 184         }
 185 #endif
 186
 187         for (;;) {
 188                 struct thread_info *context;
 189
 190                 context = (struct thread_info *)
 191                         ((unsigned long)stack & (~(THREAD_SIZE - 1)));
 192                 bp = print_context_stack(context, stack, bp, ops, data);
 193                 /*
 194                  * Should be after the line below, but somewhere
 195                  * in early boot context comes out corrupted and we
 196                  * can't reference it:
 197                  */
 198                 if (ops->stack(data, "IRQ") < 0)
 199                         break;
 200                 stack = (unsigned long *)context->previous_esp;
 201                 if (!stack)
 202                         break;
 203                 touch_nmi_watchdog();
 204         }
 205 }
 206 EXPORT_SYMBOL(dump_trace);
 207
 208 static void
 209 print_trace_warning_symbol(void *data, char *msg, unsigned long symbol)
 210 {
 211         printk(data);
 212         print_symbol(msg, symbol);
 213         printk("\n");
 214 }
 215
 216 static void print_trace_warning(void *data, char *msg)
 217 {
 218         printk("%s%s\n", (char *)data, msg);
 219 }
 220
 221 static int print_trace_stack(void *data, char *name)
 222 {
 223         return 0;
 224 }
 225
 226 /*
 227  * Print one address/symbol entries per line.
 228  */
 229 static void print_trace_address(void *data, unsigned long addr, int reliable)
 230 {
 231         printk("%s [<%08lx>] ", (char *)data, addr);
 232         if (!reliable)
 233                 printk("? ");
 234         print_symbol("%s\n", addr);
 235         touch_nmi_watchdog();
 236 }
 237
 238 static const struct stacktrace_ops print_trace_ops = {
 239         .warning = print_trace_warning,
 240         .warning_symbol = print_trace_warning_symbol,
 241         .stack = print_trace_stack,
 242         .address = print_trace_address,
 243 };
 244
 245 static void
 246 show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
 247                 unsigned long *stack, unsigned long bp, char *log_lvl)
 248 {
 249         dump_trace(task, regs, stack, bp, &print_trace_ops, log_lvl);
 250         printk("%s =======================\n", log_lvl);
 251 }
 252
 253 void show_trace(struct task_struct *task, struct pt_regs *regs,
 254                 unsigned long *stack, unsigned long bp)
 255 {
 256         show_trace_log_lvl(task, regs, stack, bp, "");
 257 }
 258
 259 static void
 260 show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 261                    unsigned long *sp, unsigned long bp, char *log_lvl)
 262 {
 263         unsigned long *stack;
 264         int i;
 265
 266         if (sp == NULL) {
 267                 if (task)
 268                         sp = (unsigned long *)task->thread.sp;
 269                 else
 270                         sp = (unsigned long *)&sp;
 271         }
 272
 273         stack = sp;
 274         for (i = 0; i < kstack_depth_to_print; i++) {
 275                 if (kstack_end(stack))
 276                         break;
 277                 if (i && ((i % 8) == 0))
 278                         printk("\n%s       ", log_lvl);
 279                 printk("%08lx ", *stack++);
 280         }
 281         printk("\n%sCall Trace:\n", log_lvl);
 282
 283         show_trace_log_lvl(task, regs, sp, bp, log_lvl);
 284 }
 285
 286 void show_stack(struct task_struct *task, unsigned long *sp)
 287 {
 288         printk("       ");
 289         show_stack_log_lvl(task, NULL, sp, 0, "");
 290 }
 291
 292 /*
 293  * The architecture-independent dump_stack generator
 294  */
 295 void dump_stack(void)
 296 {
 297         unsigned long bp = 0;
 298         unsigned long stack;
 299
 300 #ifdef CONFIG_FRAME_POINTER
 301         if (!bp)
 302                 asm("movl %%ebp, %0" : "=r" (bp):);
 303 #endif
 304
 305         printk("Pid: %d, comm: %.20s %s %s %.*s\n",
 306                 current->pid, current->comm, print_tainted(),
 307                 init_utsname()->release,
 308                 (int)strcspn(init_utsname()->version, " "),
 309                 init_utsname()->version);
 310
 311         show_trace(current, NULL, &stack, bp);
 312 }
 313
 314 EXPORT_SYMBOL(dump_stack);
 315
 316 void show_registers(struct pt_regs *regs)
 317 {
 318         int i;
 319
 320         print_modules();
 321         __show_registers(regs, 0);
 322
 323         printk(KERN_EMERG "Process %.*s (pid: %d, ti=%p task=%p task.ti=%p)",
 324                 TASK_COMM_LEN, current->comm, task_pid_nr(current),
 325                 current_thread_info(), current, task_thread_info(current));
 326         /*
 327          * When in-kernel, we also print out the stack and code at the
 328          * time of the fault..
 329          */
 330         if (!user_mode_vm(regs)) {
 331                 unsigned int code_prologue = code_bytes * 43 / 64;
 332                 unsigned int code_len = code_bytes;
 333                 unsigned char c;
 334                 u8 *ip;
 335
 336                 printk("\n" KERN_EMERG "Stack: ");
 337                 show_stack_log_lvl(NULL, regs, &regs->sp, 0, KERN_EMERG);
 338
 339                 printk(KERN_EMERG "Code: ");
 340
 341                 ip = (u8 *)regs->ip - code_prologue;
 342                 if (ip < (u8 *)PAGE_OFFSET || probe_kernel_address(ip, c)) {
 343                         /* try starting at EIP */
 344                         ip = (u8 *)regs->ip;
 345                         code_len = code_len - code_prologue + 1;
 346                 }
 347                 for (i = 0; i < code_len; i++, ip++) {
 348                         if (ip < (u8 *)PAGE_OFFSET ||
 349                                         probe_kernel_address(ip, c)) {
 350                                 printk(" Bad EIP value.");
 351                                 break;
 352                         }
 353                         if (ip == (u8 *)regs->ip)
 354                                 printk("<%02x> ", c);
 355                         else
 356                                 printk("%02x ", c);
 357                 }
 358         }
 359         printk("\n");
 360 }
 361
 362 int is_valid_bugaddr(unsigned long ip)
 363 {
 364         unsigned short ud2;
 365
 366         if (ip < PAGE_OFFSET)
 367                 return 0;
 368         if (probe_kernel_address((unsigned short *)ip, ud2))
 369                 return 0;
 370
 371         return ud2 == 0x0b0f;
 372 }
 373
 374 static raw_spinlock_t die_lock = __RAW_SPIN_LOCK_UNLOCKED;
 375 static int die_owner = -1;
 376 static unsigned int die_nest_count;
 377
 378 unsigned __kprobes long oops_begin(void)
 379 {
 380         unsigned long flags;
 381
 382         oops_enter();
 383
 384         if (die_owner != raw_smp_processor_id()) {
 385                 console_verbose();
 386                 raw_local_irq_save(flags);
 387                 __raw_spin_lock(&die_lock);
 388                 die_owner = smp_processor_id();
 389                 die_nest_count = 0;
 390                 bust_spinlocks(1);
 391         } else {
 392                 raw_local_irq_save(flags);
 393         }
 394         die_nest_count++;
 395         return flags;
 396 }
 397
 398 void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
 399 {
 400         bust_spinlocks(0);
 401         die_owner = -1;
 402         add_taint(TAINT_DIE);
 403         __raw_spin_unlock(&die_lock);
 404         raw_local_irq_restore(flags);
 405
 406         if (!regs)
 407                 return;
 408
 409         if (kexec_should_crash(current))
 410                 crash_kexec(regs);
 411
 412         if (in_interrupt())
 413                 panic("Fatal exception in interrupt");
 414
 415         if (panic_on_oops)
 416                 panic("Fatal exception");
 417
 418         oops_exit();
 419         do_exit(signr);
 420 }
 421
 422 int __kprobes __die(const char *str, struct pt_regs *regs, long err)
 423 {
 424         unsigned short ss;
 425         unsigned long sp;
 426
 427         printk(KERN_EMERG "%s: %04lx [#%d] ", str, err & 0xffff, ++die_counter);
 428 #ifdef CONFIG_PREEMPT
 429         printk("PREEMPT ");
 430 #endif
 431 #ifdef CONFIG_SMP
 432         printk("SMP ");
 433 #endif
 434 #ifdef CONFIG_DEBUG_PAGEALLOC
 435         printk("DEBUG_PAGEALLOC");
 436 #endif
 437         printk("\n");
 438         if (notify_die(DIE_OOPS, str, regs, err,
 439                         current->thread.trap_no, SIGSEGV) == NOTIFY_STOP)
 440                 return 1;
 441
 442         show_registers(regs);
 443         /* Executive summary in case the oops scrolled away */
 444         sp = (unsigned long) (&regs->sp);
 445         savesegment(ss, ss);
 446         if (user_mode(regs)) {
 447                 sp = regs->sp;
 448                 ss = regs->ss & 0xffff;
 449         }
 450         printk(KERN_EMERG "EIP: [<%08lx>] ", regs->ip);
 451         print_symbol("%s", regs->ip);
 452         printk(" SS:ESP %04x:%08lx\n", ss, sp);
 453         return 0;
 454 }
 455
 456 /*
 457  * This is gone through when something in the kernel has done something bad
 458  * and is about to be terminated:
 459  */
 460 void die(const char *str, struct pt_regs *regs, long err)
 461 {
 462         unsigned long flags = oops_begin();
 463
 464         if (die_nest_count < 3) {
 465                 report_bug(regs->ip, regs);
 466
 467                 if (__die(str, regs, err))
 468                         regs = NULL;
 469         } else {
 470                 printk(KERN_EMERG "Recursive die() failure, output suppressed\n");
 471         }
 472
 473         oops_end(flags, regs, SIGSEGV);
 474 }
 475
 476 static inline void
 477 die_if_kernel(const char *str, struct pt_regs *regs, long err)
 478 {
 479         if (!user_mode_vm(regs))
 480                 die(str, regs, err);
 481 }
 482
 483 static void __kprobes
 484 do_trap(int trapnr, int signr, char *str, int vm86, struct pt_regs *regs,
 485         long error_code, siginfo_t *info)
 486 {
 487         struct task_struct *tsk = current;
 488
 489         if (regs->flags & X86_VM_MASK) {
 490                 if (vm86)
 491                         goto vm86_trap;
 492                 goto trap_signal;
 493         }
 494
 495         if (!user_mode(regs))
 496                 goto kernel_trap;
 497
 498 trap_signal:
 499         /*
 500          * We want error_code and trap_no set for userspace faults and
 501          * kernelspace faults which result in die(), but not
 502          * kernelspace faults which are fixed up.  die() gives the
 503          * process no chance to handle the signal and notice the
 504          * kernel fault information, so that won't result in polluting
 505          * the information about previously queued, but not yet
 506          * delivered, faults.  See also do_general_protection below.
 507          */
 508         tsk->thread.error_code = error_code;
 509         tsk->thread.trap_no = trapnr;
 510
 511         if (info)
 512                 force_sig_info(signr, info, tsk);
 513         else
 514                 force_sig(signr, tsk);
 515         return;
 516
 517 kernel_trap:
 518         if (!fixup_exception(regs)) {
 519                 tsk->thread.error_code = error_code;
 520                 tsk->thread.trap_no = trapnr;
 521                 die(str, regs, error_code);
 522         }
 523         return;
 524
 525 vm86_trap:
 526         if (handle_vm86_trap((struct kernel_vm86_regs *) regs,
 527                                                 error_code, trapnr))
 528                 goto trap_signal;
 529         return;
 530 }
 531
 532 #define DO_TRAP(trapnr, signr, str, name)                               \
 533 void do_##name(struct pt_regs *regs, long error_code)                   \
 534 {                                                                       \
 535         trace_hardirqs_fixup();                                         \
 536         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 537                                                         == NOTIFY_STOP) \
 538                 return;                                                 \
 539         do_trap(trapnr, signr, str, 0, regs, error_code, NULL);         \
 540 }
 541
 542 #define DO_TRAP_INFO(trapnr, signr, str, name, sicode, siaddr, irq)     \
 543 void do_##name(struct pt_regs *regs, long error_code)                   \
 544 {                                                                       \
 545         siginfo_t info;                                                 \
 546         if (irq)                                                        \
 547                 local_irq_enable();                                     \
 548         info.si_signo = signr;                                          \
 549         info.si_errno = 0;                                              \
 550         info.si_code = sicode;                                          \
 551         info.si_addr = (void __user *)siaddr;                           \
 552         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 553                                                         == NOTIFY_STOP) \
 554                 return;                                                 \
 555         do_trap(trapnr, signr, str, 0, regs, error_code, &info);        \
 556 }
 557
 558 #define DO_VM86_TRAP(trapnr, signr, str, name)                          \
 559 void do_##name(struct pt_regs *regs, long error_code)                   \
 560 {                                                                       \
 561         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 562                                                         == NOTIFY_STOP) \
 563                 return;                                                 \
 564         do_trap(trapnr, signr, str, 1, regs, error_code, NULL);         \
 565 }
 566
 567 #define DO_VM86_TRAP_INFO(trapnr, signr, str, name, sicode, siaddr)     \
 568 void do_##name(struct pt_regs *regs, long error_code)                   \
 569 {                                                                       \
 570         siginfo_t info;                                                 \
 571         info.si_signo = signr;                                          \
 572         info.si_errno = 0;                                              \
 573         info.si_code = sicode;                                          \
 574         info.si_addr = (void __user *)siaddr;                           \
 575         trace_hardirqs_fixup();                                         \
 576         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 577                                                         == NOTIFY_STOP) \
 578                 return;                                                 \
 579         do_trap(trapnr, signr, str, 1, regs, error_code, &info);        \
 580 }
 581
 582 #define DO_ERROR(trapnr, signr, str, name)                              \
 583 void do_##name(struct pt_regs *regs, long error_code)                   \
 584 {                                                                       \
 585         trace_hardirqs_fixup();                                         \
 586         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 587                                                         == NOTIFY_STOP) \
 588                 return;                                                 \
 589         conditional_sti(regs);                                          \
 590         do_trap(trapnr, signr, str, 0, regs, error_code, NULL);         \
 591 }
 592
 593 #define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr, irq)    \
 594 void do_##name(struct pt_regs *regs, long error_code)                   \
 595 {                                                                       \
 596         siginfo_t info;                                                 \
 597         if (irq)                                                        \
 598                 local_irq_enable();                                     \
 599         info.si_signo = signr;                                          \
 600         info.si_errno = 0;                                              \
 601         info.si_code = sicode;                                          \
 602         info.si_addr = (void __user *)siaddr;                           \
 603         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 604                                                         == NOTIFY_STOP) \
 605                 return;                                                 \
 606         conditional_sti(regs);                                          \
 607         do_trap(trapnr, signr, str, 0, regs, error_code, &info);        \
 608 }
 609
 610 #define DO_VM86_ERROR(trapnr, signr, str, name)                         \
 611 void do_##name(struct pt_regs *regs, long error_code)                   \
 612 {                                                                       \
 613         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 614                                                         == NOTIFY_STOP) \
 615                 return;                                                 \
 616         conditional_sti(regs);                                          \
 617         do_trap(trapnr, signr, str, 1, regs, error_code, NULL);         \
 618 }
 619
 620 #define DO_VM86_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr)    \
 621 void do_##name(struct pt_regs *regs, long error_code)                   \
 622 {                                                                       \
 623         siginfo_t info;                                                 \
 624         info.si_signo = signr;                                          \
 625         info.si_errno = 0;                                              \
 626         info.si_code = sicode;                                          \
 627         info.si_addr = (void __user *)siaddr;                           \
 628         trace_hardirqs_fixup();                                         \
 629         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 630                                                         == NOTIFY_STOP) \
 631                 return;                                                 \
 632         conditional_sti(regs);                                          \
 633         do_trap(trapnr, signr, str, 1, regs, error_code, &info);        \
 634 }
 635
 636 DO_VM86_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip)
 637 DO_VM86_ERROR(4, SIGSEGV, "overflow", overflow)
 638 DO_VM86_ERROR(5, SIGSEGV, "bounds", bounds)
 639 DO_ERROR_INFO(6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip, 0)
 640 DO_TRAP(9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun)
 641 DO_TRAP(10, SIGSEGV, "invalid TSS", invalid_TSS)
 642 DO_TRAP(11, SIGBUS, "segment not present", segment_not_present)
 643 DO_TRAP(12, SIGBUS, "stack segment", stack_segment)
 644 DO_TRAP_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0, 0)
 645 DO_TRAP_INFO(32, SIGILL, "iret exception", iret_error, ILL_BADSTK, 0, 1)
 646
 647 void __kprobes
 648 do_general_protection(struct pt_regs *regs, long error_code)
 649 {
 650         struct task_struct *tsk;
 651         struct thread_struct *thread;
 652         struct tss_struct *tss;
 653         int cpu;
 654
 655         cpu = get_cpu();
 656         tss = &per_cpu(init_tss, cpu);
 657         thread = &current->thread;
 658
 659         /*
 660          * Perform the lazy TSS's I/O bitmap copy. If the TSS has an
 661          * invalid offset set (the LAZY one) and the faulting thread has
 662          * a valid I/O bitmap pointer, we copy the I/O bitmap in the TSS
 663          * and we set the offset field correctly. Then we let the CPU to
 664          * restart the faulting instruction.
 665          */
 666         if (tss->x86_tss.io_bitmap_base == INVALID_IO_BITMAP_OFFSET_LAZY &&
 667             thread->io_bitmap_ptr) {
 668                 memcpy(tss->io_bitmap, thread->io_bitmap_ptr,
 669                        thread->io_bitmap_max);
 670                 /*
 671                  * If the previously set map was extending to higher ports
 672                  * than the current one, pad extra space with 0xff (no access).
 673                  */
 674                 if (thread->io_bitmap_max < tss->io_bitmap_max) {
 675                         memset((char *) tss->io_bitmap +
 676                                 thread->io_bitmap_max, 0xff,
 677                                 tss->io_bitmap_max - thread->io_bitmap_max);
 678                 }
 679                 tss->io_bitmap_max = thread->io_bitmap_max;
 680                 tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
 681                 tss->io_bitmap_owner = thread;
 682                 put_cpu();
 683
 684                 return;
 685         }
 686         put_cpu();
 687
 688         if (regs->flags & X86_VM_MASK)
 689                 goto gp_in_vm86;
 690
 691         tsk = current;
 692         if (!user_mode(regs))
 693                 goto gp_in_kernel;
 694
 695         tsk->thread.error_code = error_code;
 696         tsk->thread.trap_no = 13;
 697
 698         if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
 699                         printk_ratelimit()) {
 700                 printk(KERN_INFO
 701                         "%s[%d] general protection ip:%lx sp:%lx error:%lx",
 702                         tsk->comm, task_pid_nr(tsk),
 703                         regs->ip, regs->sp, error_code);
 704                 print_vma_addr(" in ", regs->ip);
 705                 printk("\n");
 706         }
 707
 708         force_sig(SIGSEGV, tsk);
 709         return;
 710
 711 gp_in_vm86:
 712         local_irq_enable();
 713         handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
 714         return;
 715
 716 gp_in_kernel:
 717         if (fixup_exception(regs))
 718                 return;
 719
 720         tsk->thread.error_code = error_code;
 721         tsk->thread.trap_no = 13;
 722         if (notify_die(DIE_GPF, "general protection fault", regs,
 723                                 error_code, 13, SIGSEGV) == NOTIFY_STOP)
 724                 return;
 725         die("general protection fault", regs, error_code);
 726 }
 727
 728 static notrace __kprobes void
 729 mem_parity_error(unsigned char reason, struct pt_regs *regs)
 730 {
 731         printk(KERN_EMERG
 732                 "Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
 733                         reason, smp_processor_id());
 734
 735         printk(KERN_EMERG
 736                 "You have some hardware problem, likely on the PCI bus.\n");
 737
 738 #if defined(CONFIG_EDAC)
 739         if (edac_handler_set()) {
 740                 edac_atomic_assert_error();
 741                 return;
 742         }
 743 #endif
 744
 745         if (panic_on_unrecovered_nmi)
 746                 panic("NMI: Not continuing");
 747
 748         printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
 749
 750         /* Clear and disable the memory parity error line. */
 751         clear_mem_error(reason);
 752 }
 753
 754 static notrace __kprobes void
 755 io_check_error(unsigned char reason, struct pt_regs *regs)
 756 {
 757         unsigned long i;
 758
 759         printk(KERN_EMERG "NMI: IOCK error (debug interrupt?)\n");
 760         show_registers(regs);
 761
 762         /* Re-enable the IOCK line, wait for a few seconds */
 763         reason = (reason & 0xf) | 8;
 764         outb(reason, 0x61);
 765
 766         i = 2000;
 767         while (--i)
 768                 udelay(1000);
 769
 770         reason &= ~8;
 771         outb(reason, 0x61);
 772 }
 773
 774 static notrace __kprobes void
 775 unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
 776 {
 777         if (notify_die(DIE_NMIUNKNOWN, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP)
 778                 return;
 779 #ifdef CONFIG_MCA
 780         /*
 781          * Might actually be able to figure out what the guilty party
 782          * is:
 783          */
 784         if (MCA_bus) {
 785                 mca_handle_nmi();
 786                 return;
 787         }
 788 #endif
 789         printk(KERN_EMERG
 790                 "Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
 791                         reason, smp_processor_id());
 792
 793         printk(KERN_EMERG "Do you have a strange power saving mode enabled?\n");
 794         if (panic_on_unrecovered_nmi)
 795                 panic("NMI: Not continuing");
 796
 797         printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
 798 }
 799
 800 static DEFINE_SPINLOCK(nmi_print_lock);
 801
 802 void notrace __kprobes die_nmi(char *str, struct pt_regs *regs, int do_panic)
 803 {
 804         if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, 2, SIGINT) == NOTIFY_STOP)
 805                 return;
 806
 807         spin_lock(&nmi_print_lock);
 808         /*
 809         * We are in trouble anyway, lets at least try
 810         * to get a message out:
 811         */
 812         bust_spinlocks(1);
 813         printk(KERN_EMERG "%s", str);
 814         printk(" on CPU%d, ip %08lx, registers:\n",
 815                 smp_processor_id(), regs->ip);
 816         show_registers(regs);
 817         if (do_panic)
 818                 panic("Non maskable interrupt");
 819         console_silent();
 820         spin_unlock(&nmi_print_lock);
 821         bust_spinlocks(0);
 822
 823         /*
 824          * If we are in kernel we are probably nested up pretty bad
 825          * and might aswell get out now while we still can:
 826          */
 827         if (!user_mode_vm(regs)) {
 828                 current->thread.trap_no = 2;
 829                 crash_kexec(regs);
 830         }
 831
 832         do_exit(SIGSEGV);
 833 }
 834
 835 static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
 836 {
 837         unsigned char reason = 0;
 838         int cpu;
 839
 840         cpu = smp_processor_id();
 841
 842         /* Only the BSP gets external NMIs from the system. */
 843         if (!cpu)
 844                 reason = get_nmi_reason();
 845
 846         if (!(reason & 0xc0)) {
 847                 if (notify_die(DIE_NMI_IPI, "nmi_ipi", regs, reason, 2, SIGINT)
 848                                                                 == NOTIFY_STOP)
 849                         return;
 850 #ifdef CONFIG_X86_LOCAL_APIC
 851                 /*
 852                  * Ok, so this is none of the documented NMI sources,
 853                  * so it must be the NMI watchdog.
 854                  */
 855                 if (nmi_watchdog_tick(regs, reason))
 856                         return;
 857                 if (!do_nmi_callback(regs, cpu))
 858                         unknown_nmi_error(reason, regs);
 859 #else
 860                 unknown_nmi_error(reason, regs);
 861 #endif
 862
 863                 return;
 864         }
 865         if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP)
 866                 return;
 867
 868         /* AK: following checks seem to be broken on modern chipsets. FIXME */
 869         if (reason & 0x80)
 870                 mem_parity_error(reason, regs);
 871         if (reason & 0x40)
 872                 io_check_error(reason, regs);
 873         /*
 874          * Reassert NMI in case it became active meanwhile
 875          * as it's edge-triggered:
 876          */
 877         reassert_nmi();
 878 }
 879
 880 notrace __kprobes void do_nmi(struct pt_regs *regs, long error_code)
 881 {
 882         int cpu;
 883
 884         nmi_enter();
 885
 886         cpu = smp_processor_id();
 887
 888         ++nmi_count(cpu);
 889
 890         if (!ignore_nmis)
 891                 default_do_nmi(regs);
 892
 893         nmi_exit();
 894 }
 895
 896 void stop_nmi(void)
 897 {
 898         acpi_nmi_disable();
 899         ignore_nmis++;
 900 }
 901
 902 void restart_nmi(void)
 903 {
 904         ignore_nmis--;
 905         acpi_nmi_enable();
 906 }
 907
 908 void __kprobes do_int3(struct pt_regs *regs, long error_code)
 909 {
 910 #ifdef CONFIG_KPROBES
 911         trace_hardirqs_fixup();
 912
 913         if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP)
 914                         == NOTIFY_STOP)
 915                 return;
 916         /*
 917          * This is an interrupt gate, because kprobes wants interrupts
 918          * disabled. Normal trap handlers don't.
 919          */
 920         conditional_sti(regs);
 921 #else
 922         if (notify_die(DIE_TRAP, "int3", regs, error_code, 3, SIGTRAP)
 923                         == NOTIFY_STOP)
 924                 return;
 925 #endif
 926
 927         do_trap(3, SIGTRAP, "int3", 1, regs, error_code, NULL);
 928 }
 929
 930 /*
 931  * Our handling of the processor debug registers is non-trivial.
 932  * We do not clear them on entry and exit from the kernel. Therefore
 933  * it is possible to get a watchpoint trap here from inside the kernel.
 934  * However, the code in ./ptrace.c has ensured that the user can
 935  * only set watchpoints on userspace addresses. Therefore the in-kernel
 936  * watchpoint trap can only occur in code which is reading/writing
 937  * from user space. Such code must not hold kernel locks (since it
 938  * can equally take a page fault), therefore it is safe to call
 939  * force_sig_info even though that claims and releases locks.
 940  *
 941  * Code in ./signal.c ensures that the debug control register
 942  * is restored before we deliver any signal, and therefore that
 943  * user code runs with the correct debug control register even though
 944  * we clear it here.
 945  *
 946  * Being careful here means that we don't have to be as careful in a
 947  * lot of more complicated places (task switching can be a bit lazy
 948  * about restoring all the debug state, and ptrace doesn't have to
 949  * find every occurrence of the TF bit that could be saved away even
 950  * by user code)
 951  */
 952 void __kprobes do_debug(struct pt_regs *regs, long error_code)
 953 {
 954         struct task_struct *tsk = current;
 955         unsigned int condition;
 956         int si_code;
 957
 958         trace_hardirqs_fixup();
 959
 960         get_debugreg(condition, 6);
 961
 962         /*
 963          * The processor cleared BTF, so don't mark that we need it set.
 964          */
 965         clear_tsk_thread_flag(tsk, TIF_DEBUGCTLMSR);
 966         tsk->thread.debugctlmsr = 0;
 967
 968         if (notify_die(DIE_DEBUG, "debug", regs, condition, error_code,
 969                                                 SIGTRAP) == NOTIFY_STOP)
 970                 return;
 971         /* It's safe to allow irq's after DR6 has been saved */
 972         if (regs->flags & X86_EFLAGS_IF)
 973                 local_irq_enable();
 974
 975         /* Mask out spurious debug traps due to lazy DR7 setting */
 976         if (condition & (DR_TRAP0|DR_TRAP1|DR_TRAP2|DR_TRAP3)) {
 977                 if (!tsk->thread.debugreg7)
 978                         goto clear_dr7;
 979         }
 980
 981         if (regs->flags & X86_VM_MASK)
 982                 goto debug_vm86;
 983
 984         /* Save debug status register where ptrace can see it */
 985         tsk->thread.debugreg6 = condition;
 986
 987         /*
 988          * Single-stepping through TF: make sure we ignore any events in
 989          * kernel space (but re-enable TF when returning to user mode).
 990          */
 991         if (condition & DR_STEP) {
 992                 /*
 993                  * We already checked v86 mode above, so we can
 994                  * check for kernel mode by just checking the CPL
 995                  * of CS.
 996                  */
 997                 if (!user_mode(regs))
 998                         goto clear_TF_reenable;
 999         }
1000
1001         si_code = get_si_code((unsigned long)condition);
1002         /* Ok, finally something we can handle */
1003         send_sigtrap(tsk, regs, error_code, si_code);
1004
1005         /*
1006          * Disable additional traps. They'll be re-enabled when
1007          * the signal is delivered.
1008          */
1009 clear_dr7:
1010         set_debugreg(0, 7);
1011         return;
1012
1013 debug_vm86:
1014         handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code, 1);
1015         return;
1016
1017 clear_TF_reenable:
1018         set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
1019         regs->flags &= ~X86_EFLAGS_TF;
1020         return;
1021 }
1022
1023 /*
1024  * Note that we play around with the 'TS' bit in an attempt to get
1025  * the correct behaviour even in the presence of the asynchronous
1026  * IRQ13 behaviour
1027  */
1028 void math_error(void __user *ip)
1029 {
1030         struct task_struct *task;
1031         siginfo_t info;
1032         unsigned short cwd, swd;
1033
1034         /*
1035          * Save the info for the exception handler and clear the error.
1036          */
1037         task = current;
1038         save_init_fpu(task);
1039         task->thread.trap_no = 16;
1040         task->thread.error_code = 0;
1041         info.si_signo = SIGFPE;
1042         info.si_errno = 0;
1043         info.si_code = __SI_FAULT;
1044         info.si_addr = ip;
1045         /*
1046          * (~cwd & swd) will mask out exceptions that are not set to unmasked
1047          * status.  0x3f is the exception bits in these regs, 0x200 is the
1048          * C1 reg you need in case of a stack fault, 0x040 is the stack
1049          * fault bit.  We should only be taking one exception at a time,
1050          * so if this combination doesn't produce any single exception,
1051          * then we have a bad program that isn't synchronizing its FPU usage
1052          * and it will suffer the consequences since we won't be able to
1053          * fully reproduce the context of the exception
1054          */
1055         cwd = get_fpu_cwd(task);
1056         swd = get_fpu_swd(task);
1057         switch (swd & ~cwd & 0x3f) {
1058         case 0x000: /* No unmasked exception */
1059                 return;
1060         default: /* Multiple exceptions */
1061                 break;
1062         case 0x001: /* Invalid Op */
1063                 /*
1064                  * swd & 0x240 == 0x040: Stack Underflow
1065                  * swd & 0x240 == 0x240: Stack Overflow
1066                  * User must clear the SF bit (0x40) if set
1067                  */
1068                 info.si_code = FPE_FLTINV;
1069                 break;
1070         case 0x002: /* Denormalize */
1071         case 0x010: /* Underflow */
1072                 info.si_code = FPE_FLTUND;
1073                 break;
1074         case 0x004: /* Zero Divide */
1075                 info.si_code = FPE_FLTDIV;
1076                 break;
1077         case 0x008: /* Overflow */
1078                 info.si_code = FPE_FLTOVF;
1079                 break;
1080         case 0x020: /* Precision */
1081                 info.si_code = FPE_FLTRES;
1082                 break;
1083         }
1084         force_sig_info(SIGFPE, &info, task);
1085 }
1086
1087 void do_coprocessor_error(struct pt_regs *regs, long error_code)
1088 {
1089         ignore_fpu_irq = 1;
1090         math_error((void __user *)regs->ip);
1091 }
1092
1093 static void simd_math_error(void __user *ip)
1094 {
1095         struct task_struct *task;
1096         siginfo_t info;
1097         unsigned short mxcsr;
1098
1099         /*
1100          * Save the info for the exception handler and clear the error.
1101          */
1102         task = current;
1103         save_init_fpu(task);
1104         task->thread.trap_no = 19;
1105         task->thread.error_code = 0;
1106         info.si_signo = SIGFPE;
1107         info.si_errno = 0;
1108         info.si_code = __SI_FAULT;
1109         info.si_addr = ip;
1110         /*
1111          * The SIMD FPU exceptions are handled a little differently, as there
1112          * is only a single status/control register.  Thus, to determine which
1113          * unmasked exception was caught we must mask the exception mask bits
1114          * at 0x1f80, and then use these to mask the exception bits at 0x3f.
1115          */
1116         mxcsr = get_fpu_mxcsr(task);
1117         switch (~((mxcsr & 0x1f80) >> 7) & (mxcsr & 0x3f)) {
1118         case 0x000:
1119         default:
1120                 break;
1121         case 0x001: /* Invalid Op */
1122                 info.si_code = FPE_FLTINV;
1123                 break;
1124         case 0x002: /* Denormalize */
1125         case 0x010: /* Underflow */
1126                 info.si_code = FPE_FLTUND;
1127                 break;
1128         case 0x004: /* Zero Divide */
1129                 info.si_code = FPE_FLTDIV;
1130                 break;
1131         case 0x008: /* Overflow */
1132                 info.si_code = FPE_FLTOVF;
1133                 break;
1134         case 0x020: /* Precision */
1135                 info.si_code = FPE_FLTRES;
1136                 break;
1137         }
1138         force_sig_info(SIGFPE, &info, task);
1139 }
1140
1141 void do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
1142 {
1143         if (cpu_has_xmm) {
1144                 /* Handle SIMD FPU exceptions on PIII+ processors. */
1145                 ignore_fpu_irq = 1;
1146                 simd_math_error((void __user *)regs->ip);
1147                 return;
1148         }
1149         /*
1150          * Handle strange cache flush from user space exception
1151          * in all other cases.  This is undocumented behaviour.
1152          */
1153         if (regs->flags & X86_VM_MASK) {
1154                 handle_vm86_fault((struct kernel_vm86_regs *)regs, error_code);
1155                 return;
1156         }
1157         current->thread.trap_no = 19;
1158         current->thread.error_code = error_code;
1159         die_if_kernel("cache flush denied", regs, error_code);
1160         force_sig(SIGSEGV, current);
1161 }
1162
1163 void do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
1164 {
1165 #if 0
1166         /* No need to warn about this any longer. */
1167         printk(KERN_INFO "Ignoring P6 Local APIC Spurious Interrupt Bug...\n");
1168 #endif
1169 }
1170
1171 unsigned long patch_espfix_desc(unsigned long uesp, unsigned long kesp)
1172 {
1173         struct desc_struct *gdt = get_cpu_gdt_table(smp_processor_id());
1174         unsigned long base = (kesp - uesp) & -THREAD_SIZE;
1175         unsigned long new_kesp = kesp - base;
1176         unsigned long lim_pages = (new_kesp | (THREAD_SIZE - 1)) >> PAGE_SHIFT;
1177         __u64 desc = *(__u64 *)&gdt[GDT_ENTRY_ESPFIX_SS];
1178
1179         /* Set up base for espfix segment */
1180         desc &= 0x00f0ff0000000000ULL;
1181         desc |= ((((__u64)base) << 16) & 0x000000ffffff0000ULL) |
1182                 ((((__u64)base) << 32) & 0xff00000000000000ULL) |
1183                 ((((__u64)lim_pages) << 32) & 0x000f000000000000ULL) |
1184                 (lim_pages & 0xffff);
1185         *(__u64 *)&gdt[GDT_ENTRY_ESPFIX_SS] = desc;
1186
1187         return new_kesp;
1188 }
1189
1190 /*
1191  * 'math_state_restore()' saves the current math information in the
1192  * old math state array, and gets the new ones from the current task
1193  *
1194  * Careful.. There are problems with IBM-designed IRQ13 behaviour.
1195  * Don't touch unless you *really* know how it works.
1196  *
1197  * Must be called with kernel preemption disabled (in this case,
1198  * local interrupts are disabled at the call-site in entry.S).
1199  */
1200 asmlinkage void math_state_restore(void)
1201 {
1202         struct thread_info *thread = current_thread_info();
1203         struct task_struct *tsk = thread->task;
1204
1205         if (!tsk_used_math(tsk)) {
1206                 local_irq_enable();
1207                 /*
1208                  * does a slab alloc which can sleep
1209                  */
1210                 if (init_fpu(tsk)) {
1211                         /*
1212                          * ran out of memory!
1213                          */
1214                         do_group_exit(SIGKILL);
1215                         return;
1216                 }
1217                 local_irq_disable();
1218         }
1219
1220         clts();                         /* Allow maths ops (or we recurse) */
1221         restore_fpu(tsk);
1222         thread->status |= TS_USEDFPU;   /* So we fnsave on switch_to() */
1223         tsk->fpu_counter++;
1224 }
1225 EXPORT_SYMBOL_GPL(math_state_restore);
1226
1227 #ifndef CONFIG_MATH_EMULATION
1228
1229 asmlinkage void math_emulate(long arg)
1230 {
1231         printk(KERN_EMERG
1232                 "math-emulation not enabled and no coprocessor found.\n");
1233         printk(KERN_EMERG "killing %s.\n", current->comm);
1234         force_sig(SIGFPE, current);
1235         schedule();
1236 }
1237
1238 #endif /* CONFIG_MATH_EMULATION */
1239
1240 void __kprobes do_device_not_available(struct pt_regs *regs, long error)
1241 {
1242         if (read_cr0() & X86_CR0_EM) {
1243                 conditional_sti(regs);
1244                 math_emulate(0);
1245         } else {
1246                 math_state_restore(); /* interrupts still off */
1247                 conditional_sti(regs);
1248         }
1249 }
1250
1251 void __init trap_init(void)
1252 {
1253         int i;
1254
1255 #ifdef CONFIG_EISA
1256         void __iomem *p = early_ioremap(0x0FFFD9, 4);
1257
1258         if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24))
1259                 EISA_bus = 1;
1260         early_iounmap(p, 4);
1261 #endif
1262
1263         set_intr_gate(0, &divide_error);
1264         set_intr_gate(1, &debug);
1265         set_intr_gate(2, &nmi);
1266         set_system_intr_gate(3, &int3); /* int3 can be called from all */
1267         set_system_intr_gate(4, &overflow); /* int4 can be called from all */
1268         set_intr_gate(5, &bounds);
1269         set_intr_gate(6, &invalid_op);
1270         set_intr_gate(7, &device_not_available);
1271         set_task_gate(8, GDT_ENTRY_DOUBLEFAULT_TSS);
1272         set_trap_gate(9, &coprocessor_segment_overrun);
1273         set_trap_gate(10, &invalid_TSS);
1274         set_trap_gate(11, &segment_not_present);
1275         set_trap_gate(12, &stack_segment);
1276         set_trap_gate(13, &general_protection);
1277         set_intr_gate(14, &page_fault);
1278         set_trap_gate(15, &spurious_interrupt_bug);
1279         set_trap_gate(16, &coprocessor_error);
1280         set_trap_gate(17, &alignment_check);
1281 #ifdef CONFIG_X86_MCE
1282         set_trap_gate(18, &machine_check);
1283 #endif
1284         set_trap_gate(19, &simd_coprocessor_error);
1285
1286         if (cpu_has_fxsr) {
1287                 printk(KERN_INFO "Enabling fast FPU save and restore... ");
1288                 set_in_cr4(X86_CR4_OSFXSR);
1289                 printk("done.\n");
1290         }
1291         if (cpu_has_xmm) {
1292                 printk(KERN_INFO
1293                         "Enabling unmasked SIMD FPU exception support... ");
1294                 set_in_cr4(X86_CR4_OSXMMEXCPT);
1295                 printk("done.\n");
1296         }
1297
1298         set_system_gate(SYSCALL_VECTOR, &system_call);
1299
1300         /* Reserve all the builtin and the syscall vector: */
1301         for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
1302                 set_bit(i, used_vectors);
1303
1304         set_bit(SYSCALL_VECTOR, used_vectors);
1305
1306         /*
1307          * Should be a barrier for any external CPU state:
1308          */
1309         cpu_init();
1310
1311         trap_init_hook();
1312 }
1313
1314 static int __init kstack_setup(char *s)
1315 {
1316         kstack_depth_to_print = simple_strtoul(s, NULL, 0);
1317
1318         return 1;
1319 }
1320 __setup("kstack=", kstack_setup);
1321
1322 static int __init code_bytes_setup(char *s)
1323 {
1324         code_bytes = simple_strtoul(s, NULL, 0);
1325         if (code_bytes > 8192)
1326                 code_bytes = 8192;
1327
1328         return 1;
1329 }
1330 __setup("code_bytes=", code_bytes_setup);