arch/ia64/kernel/unwind.c

   1 /*
   2  * Copyright (C) 1999-2004 Hewlett-Packard Co
   3  *      David Mosberger-Tang <davidm@hpl.hp.com>
   4  * Copyright (C) 2003 Fenghua Yu <fenghua.yu@intel.com>
   5  *      - Change pt_regs_off() to make it less dependent on pt_regs structure.
   6  */
   7 /*
   8  * This file implements call frame unwind support for the Linux
   9  * kernel.  Parsing and processing the unwind information is
  10  * time-consuming, so this implementation translates the unwind
  11  * descriptors into unwind scripts.  These scripts are very simple
  12  * (basically a sequence of assignments) and efficient to execute.
  13  * They are cached for later re-use.  Each script is specific for a
  14  * given instruction pointer address and the set of predicate values
  15  * that the script depends on (most unwind descriptors are
  16  * unconditional and scripts often do not depend on predicates at
  17  * all).  This code is based on the unwind conventions described in
  18  * the "IA-64 Software Conventions and Runtime Architecture" manual.
  19  *
  20  * SMP conventions:
  21  *      o updates to the global unwind data (in structure "unw") are serialized
  22  *        by the unw.lock spinlock
  23  *      o each unwind script has its own read-write lock; a thread must acquire
  24  *        a read lock before executing a script and must acquire a write lock
  25  *        before modifying a script
  26  *      o if both the unw.lock spinlock and a script's read-write lock must be
  27  *        acquired, then the read-write lock must be acquired first.
  28  */
  29 #include <linux/module.h>
  30 #include <linux/bootmem.h>
  31 #include <linux/elf.h>
  32 #include <linux/kernel.h>
  33 #include <linux/sched.h>
  34 #include <linux/slab.h>
  35
  36 #include <asm/unwind.h>
  37
  38 #include <asm/delay.h>
  39 #include <asm/page.h>
  40 #include <asm/ptrace.h>
  41 #include <asm/ptrace_offsets.h>
  42 #include <asm/rse.h>
  43 #include <asm/sections.h>
  44 #include <asm/system.h>
  45 #include <asm/uaccess.h>
  46
  47 #include "entry.h"
  48 #include "unwind_i.h"
  49
  50 #define UNW_LOG_CACHE_SIZE      7       /* each unw_script is ~256 bytes in size */
  51 #define UNW_CACHE_SIZE          (1 << UNW_LOG_CACHE_SIZE)
  52
  53 #define UNW_LOG_HASH_SIZE       (UNW_LOG_CACHE_SIZE + 1)
  54 #define UNW_HASH_SIZE           (1 << UNW_LOG_HASH_SIZE)
  55
  56 #define UNW_STATS       0       /* WARNING: this disabled interrupts for long time-spans!! */
  57
  58 #ifdef UNW_DEBUG
  59   static unsigned int unw_debug_level = UNW_DEBUG;
  60 #  define UNW_DEBUG_ON(n)       unw_debug_level >= n
  61    /* Do not code a printk level, not all debug lines end in newline */
  62 #  define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
  63 #  undef inline
  64 #  define inline
  65 #else /* !UNW_DEBUG */
  66 #  define UNW_DEBUG_ON(n)  0
  67 #  define UNW_DPRINT(n, ...)
  68 #endif /* UNW_DEBUG */
  69
  70 #if UNW_STATS
  71 # define STAT(x...)     x
  72 #else
  73 # define STAT(x...)
  74 #endif
  75
  76 #define alloc_reg_state()       kmalloc(sizeof(struct unw_reg_state), GFP_ATOMIC)
  77 #define free_reg_state(usr)     kfree(usr)
  78 #define alloc_labeled_state()   kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
  79 #define free_labeled_state(usr) kfree(usr)
  80
  81 typedef unsigned long unw_word;
  82 typedef unsigned char unw_hash_index_t;
  83
  84 static struct {
  85         spinlock_t lock;                        /* spinlock for unwind data */
  86
  87         /* list of unwind tables (one per load-module) */
  88         struct unw_table *tables;
  89
  90         unsigned long r0;                       /* constant 0 for r0 */
  91
  92         /* table of registers that prologues can save (and order in which they're saved): */
  93         const unsigned char save_order[8];
  94
  95         /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
  96         unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
  97
  98         unsigned short lru_head;                /* index of lead-recently used script */
  99         unsigned short lru_tail;                /* index of most-recently used script */
 100
 101         /* index into unw_frame_info for preserved register i */
 102         unsigned short preg_index[UNW_NUM_REGS];
 103
 104         short pt_regs_offsets[32];
 105
 106         /* unwind table for the kernel: */
 107         struct unw_table kernel_table;
 108
 109         /* unwind table describing the gate page (kernel code that is mapped into user space): */
 110         size_t gate_table_size;
 111         unsigned long *gate_table;
 112
 113         /* hash table that maps instruction pointer to script index: */
 114         unsigned short hash[UNW_HASH_SIZE];
 115
 116         /* script cache: */
 117         struct unw_script cache[UNW_CACHE_SIZE];
 118
 119 # ifdef UNW_DEBUG
 120         const char *preg_name[UNW_NUM_REGS];
 121 # endif
 122 # if UNW_STATS
 123         struct {
 124                 struct {
 125                         int lookups;
 126                         int hinted_hits;
 127                         int normal_hits;
 128                         int collision_chain_traversals;
 129                 } cache;
 130                 struct {
 131                         unsigned long build_time;
 132                         unsigned long run_time;
 133                         unsigned long parse_time;
 134                         int builds;
 135                         int news;
 136                         int collisions;
 137                         int runs;
 138                 } script;
 139                 struct {
 140                         unsigned long init_time;
 141                         unsigned long unwind_time;
 142                         int inits;
 143                         int unwinds;
 144                 } api;
 145         } stat;
 146 # endif
 147 } unw = {
 148         .tables = &unw.kernel_table,
 149         .lock = __SPIN_LOCK_UNLOCKED(unw.lock),
 150         .save_order = {
 151                 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
 152                 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
 153         },
 154         .preg_index = {
 155                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_GR */
 156                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_MEM */
 157                 offsetof(struct unw_frame_info, bsp_loc)/8,
 158                 offsetof(struct unw_frame_info, bspstore_loc)/8,
 159                 offsetof(struct unw_frame_info, pfs_loc)/8,
 160                 offsetof(struct unw_frame_info, rnat_loc)/8,
 161                 offsetof(struct unw_frame_info, psp)/8,
 162                 offsetof(struct unw_frame_info, rp_loc)/8,
 163                 offsetof(struct unw_frame_info, r4)/8,
 164                 offsetof(struct unw_frame_info, r5)/8,
 165                 offsetof(struct unw_frame_info, r6)/8,
 166                 offsetof(struct unw_frame_info, r7)/8,
 167                 offsetof(struct unw_frame_info, unat_loc)/8,
 168                 offsetof(struct unw_frame_info, pr_loc)/8,
 169                 offsetof(struct unw_frame_info, lc_loc)/8,
 170                 offsetof(struct unw_frame_info, fpsr_loc)/8,
 171                 offsetof(struct unw_frame_info, b1_loc)/8,
 172                 offsetof(struct unw_frame_info, b2_loc)/8,
 173                 offsetof(struct unw_frame_info, b3_loc)/8,
 174                 offsetof(struct unw_frame_info, b4_loc)/8,
 175                 offsetof(struct unw_frame_info, b5_loc)/8,
 176                 offsetof(struct unw_frame_info, f2_loc)/8,
 177                 offsetof(struct unw_frame_info, f3_loc)/8,
 178                 offsetof(struct unw_frame_info, f4_loc)/8,
 179                 offsetof(struct unw_frame_info, f5_loc)/8,
 180                 offsetof(struct unw_frame_info, fr_loc[16 - 16])/8,
 181                 offsetof(struct unw_frame_info, fr_loc[17 - 16])/8,
 182                 offsetof(struct unw_frame_info, fr_loc[18 - 16])/8,
 183                 offsetof(struct unw_frame_info, fr_loc[19 - 16])/8,
 184                 offsetof(struct unw_frame_info, fr_loc[20 - 16])/8,
 185                 offsetof(struct unw_frame_info, fr_loc[21 - 16])/8,
 186                 offsetof(struct unw_frame_info, fr_loc[22 - 16])/8,
 187                 offsetof(struct unw_frame_info, fr_loc[23 - 16])/8,
 188                 offsetof(struct unw_frame_info, fr_loc[24 - 16])/8,
 189                 offsetof(struct unw_frame_info, fr_loc[25 - 16])/8,
 190                 offsetof(struct unw_frame_info, fr_loc[26 - 16])/8,
 191                 offsetof(struct unw_frame_info, fr_loc[27 - 16])/8,
 192                 offsetof(struct unw_frame_info, fr_loc[28 - 16])/8,
 193                 offsetof(struct unw_frame_info, fr_loc[29 - 16])/8,
 194                 offsetof(struct unw_frame_info, fr_loc[30 - 16])/8,
 195                 offsetof(struct unw_frame_info, fr_loc[31 - 16])/8,
 196         },
 197         .pt_regs_offsets = {
 198                 [0] = -1,
 199                 offsetof(struct pt_regs,  r1),
 200                 offsetof(struct pt_regs,  r2),
 201                 offsetof(struct pt_regs,  r3),
 202                 [4] = -1, [5] = -1, [6] = -1, [7] = -1,
 203                 offsetof(struct pt_regs,  r8),
 204                 offsetof(struct pt_regs,  r9),
 205                 offsetof(struct pt_regs, r10),
 206                 offsetof(struct pt_regs, r11),
 207                 offsetof(struct pt_regs, r12),
 208                 offsetof(struct pt_regs, r13),
 209                 offsetof(struct pt_regs, r14),
 210                 offsetof(struct pt_regs, r15),
 211                 offsetof(struct pt_regs, r16),
 212                 offsetof(struct pt_regs, r17),
 213                 offsetof(struct pt_regs, r18),
 214                 offsetof(struct pt_regs, r19),
 215                 offsetof(struct pt_regs, r20),
 216                 offsetof(struct pt_regs, r21),
 217                 offsetof(struct pt_regs, r22),
 218                 offsetof(struct pt_regs, r23),
 219                 offsetof(struct pt_regs, r24),
 220                 offsetof(struct pt_regs, r25),
 221                 offsetof(struct pt_regs, r26),
 222                 offsetof(struct pt_regs, r27),
 223                 offsetof(struct pt_regs, r28),
 224                 offsetof(struct pt_regs, r29),
 225                 offsetof(struct pt_regs, r30),
 226                 offsetof(struct pt_regs, r31),
 227         },
 228         .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
 229 #ifdef UNW_DEBUG
 230         .preg_name = {
 231                 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
 232                 "r4", "r5", "r6", "r7",
 233                 "ar.unat", "pr", "ar.lc", "ar.fpsr",
 234                 "b1", "b2", "b3", "b4", "b5",
 235                 "f2", "f3", "f4", "f5",
 236                 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
 237                 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
 238         }
 239 #endif
 240 };
 241
 242 static inline int
 243 read_only (void *addr)
 244 {
 245         return (unsigned long) ((char *) addr - (char *) &unw.r0) < sizeof(unw.r0);
 246 }
 247
 248 /*
 249  * Returns offset of rREG in struct pt_regs.
 250  */
 251 static inline unsigned long
 252 pt_regs_off (unsigned long reg)
 253 {
 254         short off = -1;
 255
 256         if (reg < ARRAY_SIZE(unw.pt_regs_offsets))
 257                 off = unw.pt_regs_offsets[reg];
 258
 259         if (off < 0) {
 260                 UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __func__, reg);
 261                 off = 0;
 262         }
 263         return (unsigned long) off;
 264 }
 265
 266 static inline struct pt_regs *
 267 get_scratch_regs (struct unw_frame_info *info)
 268 {
 269         if (!info->pt) {
 270                 /* This should not happen with valid unwind info.  */
 271                 UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __func__);
 272                 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
 273                         info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
 274                 else
 275                         info->pt = info->sp - 16;
 276         }
 277         UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __func__, info->sp, info->pt);
 278         return (struct pt_regs *) info->pt;
 279 }
 280
 281 /* Unwind accessors.  */
 282
 283 int
 284 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
 285 {
 286         unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
 287         struct unw_ireg *ireg;
 288         struct pt_regs *pt;
 289
 290         if ((unsigned) regnum - 1 >= 127) {
 291                 if (regnum == 0 && !write) {
 292                         *val = 0;       /* read r0 always returns 0 */
 293                         *nat = 0;
 294                         return 0;
 295                 }
 296                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
 297                            __func__, regnum);
 298                 return -1;
 299         }
 300
 301         if (regnum < 32) {
 302                 if (regnum >= 4 && regnum <= 7) {
 303                         /* access a preserved register */
 304                         ireg = &info->r4 + (regnum - 4);
 305                         addr = ireg->loc;
 306                         if (addr) {
 307                                 nat_addr = addr + ireg->nat.off;
 308                                 switch (ireg->nat.type) {
 309                                       case UNW_NAT_VAL:
 310                                         /* simulate getf.sig/setf.sig */
 311                                         if (write) {
 312                                                 if (*nat) {
 313                                                         /* write NaTVal and be done with it */
 314                                                         addr[0] = 0;
 315                                                         addr[1] = 0x1fffe;
 316                                                         return 0;
 317                                                 }
 318                                                 addr[1] = 0x1003e;
 319                                         } else {
 320                                                 if (addr[0] == 0 && addr[1] == 0x1ffe) {
 321                                                         /* return NaT and be done with it */
 322                                                         *val = 0;
 323                                                         *nat = 1;
 324                                                         return 0;
 325                                                 }
 326                                         }
 327                                         /* fall through */
 328                                       case UNW_NAT_NONE:
 329                                         dummy_nat = 0;
 330                                         nat_addr = &dummy_nat;
 331                                         break;
 332
 333                                       case UNW_NAT_MEMSTK:
 334                                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 335                                         break;
 336
 337                                       case UNW_NAT_REGSTK:
 338                                         nat_addr = ia64_rse_rnat_addr(addr);
 339                                         if ((unsigned long) addr < info->regstk.limit
 340                                             || (unsigned long) addr >= info->regstk.top)
 341                                         {
 342                                                 UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
 343                                                         "[0x%lx-0x%lx)\n",
 344                                                         __func__, (void *) addr,
 345                                                         info->regstk.limit,
 346                                                         info->regstk.top);
 347                                                 return -1;
 348                                         }
 349                                         if ((unsigned long) nat_addr >= info->regstk.top)
 350                                                 nat_addr = &info->sw->ar_rnat;
 351                                         nat_mask = (1UL << ia64_rse_slot_num(addr));
 352                                         break;
 353                                 }
 354                         } else {
 355                                 addr = &info->sw->r4 + (regnum - 4);
 356                                 nat_addr = &info->sw->ar_unat;
 357                                 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 358                         }
 359                 } else {
 360                         /* access a scratch register */
 361                         pt = get_scratch_regs(info);
 362                         addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
 363                         if (info->pri_unat_loc)
 364                                 nat_addr = info->pri_unat_loc;
 365                         else
 366                                 nat_addr = &info->sw->caller_unat;
 367                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 368                 }
 369         } else {
 370                 /* access a stacked register */
 371                 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
 372                 nat_addr = ia64_rse_rnat_addr(addr);
 373                 if ((unsigned long) addr < info->regstk.limit
 374                     || (unsigned long) addr >= info->regstk.top)
 375                 {
 376                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
 377                                    "of rbs\n",  __func__);
 378                         return -1;
 379                 }
 380                 if ((unsigned long) nat_addr >= info->regstk.top)
 381                         nat_addr = &info->sw->ar_rnat;
 382                 nat_mask = (1UL << ia64_rse_slot_num(addr));
 383         }
 384
 385         if (write) {
 386                 if (read_only(addr)) {
 387                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 388                                 __func__);
 389                 } else {
 390                         *addr = *val;
 391                         if (*nat)
 392                                 *nat_addr |= nat_mask;
 393                         else
 394                                 *nat_addr &= ~nat_mask;
 395                 }
 396         } else {
 397                 if ((*nat_addr & nat_mask) == 0) {
 398                         *val = *addr;
 399                         *nat = 0;
 400                 } else {
 401                         *val = 0;       /* if register is a NaT, *addr may contain kernel data! */
 402                         *nat = 1;
 403                 }
 404         }
 405         return 0;
 406 }
 407 EXPORT_SYMBOL(unw_access_gr);
 408
 409 int
 410 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 411 {
 412         unsigned long *addr;
 413         struct pt_regs *pt;
 414
 415         switch (regnum) {
 416                 /* scratch: */
 417               case 0: pt = get_scratch_regs(info); addr = &pt->b0; break;
 418               case 6: pt = get_scratch_regs(info); addr = &pt->b6; break;
 419               case 7: pt = get_scratch_regs(info); addr = &pt->b7; break;
 420
 421                 /* preserved: */
 422               case 1: case 2: case 3: case 4: case 5:
 423                 addr = *(&info->b1_loc + (regnum - 1));
 424                 if (!addr)
 425                         addr = &info->sw->b1 + (regnum - 1);
 426                 break;
 427
 428               default:
 429                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
 430                            __func__, regnum);
 431                 return -1;
 432         }
 433         if (write)
 434                 if (read_only(addr)) {
 435                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 436                                 __func__);
 437                 } else
 438                         *addr = *val;
 439         else
 440                 *val = *addr;
 441         return 0;
 442 }
 443 EXPORT_SYMBOL(unw_access_br);
 444
 445 int
 446 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
 447 {
 448         struct ia64_fpreg *addr = NULL;
 449         struct pt_regs *pt;
 450
 451         if ((unsigned) (regnum - 2) >= 126) {
 452                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
 453                            __func__, regnum);
 454                 return -1;
 455         }
 456
 457         if (regnum <= 5) {
 458                 addr = *(&info->f2_loc + (regnum - 2));
 459                 if (!addr)
 460                         addr = &info->sw->f2 + (regnum - 2);
 461         } else if (regnum <= 15) {
 462                 if (regnum <= 11) {
 463                         pt = get_scratch_regs(info);
 464                         addr = &pt->f6  + (regnum - 6);
 465                 }
 466                 else
 467                         addr = &info->sw->f12 + (regnum - 12);
 468         } else if (regnum <= 31) {
 469                 addr = info->fr_loc[regnum - 16];
 470                 if (!addr)
 471                         addr = &info->sw->f16 + (regnum - 16);
 472         } else {
 473                 struct task_struct *t = info->task;
 474
 475                 if (write)
 476                         ia64_sync_fph(t);
 477                 else
 478                         ia64_flush_fph(t);
 479                 addr = t->thread.fph + (regnum - 32);
 480         }
 481
 482         if (write)
 483                 if (read_only(addr)) {
 484                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 485                                 __func__);
 486                 } else
 487                         *addr = *val;
 488         else
 489                 *val = *addr;
 490         return 0;
 491 }
 492 EXPORT_SYMBOL(unw_access_fr);
 493
 494 int
 495 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 496 {
 497         unsigned long *addr;
 498         struct pt_regs *pt;
 499
 500         switch (regnum) {
 501               case UNW_AR_BSP:
 502                 addr = info->bsp_loc;
 503                 if (!addr)
 504                         addr = &info->sw->ar_bspstore;
 505                 break;
 506
 507               case UNW_AR_BSPSTORE:
 508                 addr = info->bspstore_loc;
 509                 if (!addr)
 510                         addr = &info->sw->ar_bspstore;
 511                 break;
 512
 513               case UNW_AR_PFS:
 514                 addr = info->pfs_loc;
 515                 if (!addr)
 516                         addr = &info->sw->ar_pfs;
 517                 break;
 518
 519               case UNW_AR_RNAT:
 520                 addr = info->rnat_loc;
 521                 if (!addr)
 522                         addr = &info->sw->ar_rnat;
 523                 break;
 524
 525               case UNW_AR_UNAT:
 526                 addr = info->unat_loc;
 527                 if (!addr)
 528                         addr = &info->sw->caller_unat;
 529                 break;
 530
 531               case UNW_AR_LC:
 532                 addr = info->lc_loc;
 533                 if (!addr)
 534                         addr = &info->sw->ar_lc;
 535                 break;
 536
 537               case UNW_AR_EC:
 538                 if (!info->cfm_loc)
 539                         return -1;
 540                 if (write)
 541                         *info->cfm_loc =
 542                                 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
 543                 else
 544                         *val = (*info->cfm_loc >> 52) & 0x3f;
 545                 return 0;
 546
 547               case UNW_AR_FPSR:
 548                 addr = info->fpsr_loc;
 549                 if (!addr)
 550                         addr = &info->sw->ar_fpsr;
 551                 break;
 552
 553               case UNW_AR_RSC:
 554                 pt = get_scratch_regs(info);
 555                 addr = &pt->ar_rsc;
 556                 break;
 557
 558               case UNW_AR_CCV:
 559                 pt = get_scratch_regs(info);
 560                 addr = &pt->ar_ccv;
 561                 break;
 562
 563               case UNW_AR_CSD:
 564                 pt = get_scratch_regs(info);
 565                 addr = &pt->ar_csd;
 566                 break;
 567
 568               case UNW_AR_SSD:
 569                 pt = get_scratch_regs(info);
 570                 addr = &pt->ar_ssd;
 571                 break;
 572
 573               default:
 574                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
 575                            __func__, regnum);
 576                 return -1;
 577         }
 578
 579         if (write) {
 580                 if (read_only(addr)) {
 581                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 582                                 __func__);
 583                 } else
 584                         *addr = *val;
 585         } else
 586                 *val = *addr;
 587         return 0;
 588 }
 589 EXPORT_SYMBOL(unw_access_ar);
 590
 591 int
 592 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
 593 {
 594         unsigned long *addr;
 595
 596         addr = info->pr_loc;
 597         if (!addr)
 598                 addr = &info->sw->pr;
 599
 600         if (write) {
 601                 if (read_only(addr)) {
 602                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 603                                 __func__);
 604                 } else
 605                         *addr = *val;
 606         } else
 607                 *val = *addr;
 608         return 0;
 609 }
 610 EXPORT_SYMBOL(unw_access_pr);
 611
 612 \f
 613 /* Routines to manipulate the state stack.  */
 614
 615 static inline void
 616 push (struct unw_state_record *sr)
 617 {
 618         struct unw_reg_state *rs;
 619
 620         rs = alloc_reg_state();
 621         if (!rs) {
 622                 printk(KERN_ERR "unwind: cannot stack reg state!\n");
 623                 return;
 624         }
 625         memcpy(rs, &sr->curr, sizeof(*rs));
 626         sr->curr.next = rs;
 627 }
 628
 629 static void
 630 pop (struct unw_state_record *sr)
 631 {
 632         struct unw_reg_state *rs = sr->curr.next;
 633
 634         if (!rs) {
 635                 printk(KERN_ERR "unwind: stack underflow!\n");
 636                 return;
 637         }
 638         memcpy(&sr->curr, rs, sizeof(*rs));
 639         free_reg_state(rs);
 640 }
 641
 642 /* Make a copy of the state stack.  Non-recursive to avoid stack overflows.  */
 643 static struct unw_reg_state *
 644 dup_state_stack (struct unw_reg_state *rs)
 645 {
 646         struct unw_reg_state *copy, *prev = NULL, *first = NULL;
 647
 648         while (rs) {
 649                 copy = alloc_reg_state();
 650                 if (!copy) {
 651                         printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
 652                         return NULL;
 653                 }
 654                 memcpy(copy, rs, sizeof(*copy));
 655                 if (first)
 656                         prev->next = copy;
 657                 else
 658                         first = copy;
 659                 rs = rs->next;
 660                 prev = copy;
 661         }
 662         return first;
 663 }
 664
 665 /* Free all stacked register states (but not RS itself).  */
 666 static void
 667 free_state_stack (struct unw_reg_state *rs)
 668 {
 669         struct unw_reg_state *p, *next;
 670
 671         for (p = rs->next; p != NULL; p = next) {
 672                 next = p->next;
 673                 free_reg_state(p);
 674         }
 675         rs->next = NULL;
 676 }
 677 \f
 678 /* Unwind decoder routines */
 679
 680 static enum unw_register_index __attribute_const__
 681 decode_abreg (unsigned char abreg, int memory)
 682 {
 683         switch (abreg) {
 684               case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
 685               case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
 686               case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
 687               case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
 688               case 0x60: return UNW_REG_PR;
 689               case 0x61: return UNW_REG_PSP;
 690               case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
 691               case 0x63: return UNW_REG_RP;
 692               case 0x64: return UNW_REG_BSP;
 693               case 0x65: return UNW_REG_BSPSTORE;
 694               case 0x66: return UNW_REG_RNAT;
 695               case 0x67: return UNW_REG_UNAT;
 696               case 0x68: return UNW_REG_FPSR;
 697               case 0x69: return UNW_REG_PFS;
 698               case 0x6a: return UNW_REG_LC;
 699               default:
 700                 break;
 701         }
 702         UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __func__, abreg);
 703         return UNW_REG_LC;
 704 }
 705
 706 static void
 707 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
 708 {
 709         reg->val = val;
 710         reg->where = where;
 711         if (reg->when == UNW_WHEN_NEVER)
 712                 reg->when = when;
 713 }
 714
 715 static void
 716 alloc_spill_area (unsigned long *offp, unsigned long regsize,
 717                   struct unw_reg_info *lo, struct unw_reg_info *hi)
 718 {
 719         struct unw_reg_info *reg;
 720
 721         for (reg = hi; reg >= lo; --reg) {
 722                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 723                         reg->where = UNW_WHERE_PSPREL;
 724                         *offp -= regsize;
 725                         reg->val = *offp;
 726                 }
 727         }
 728 }
 729
 730 static inline void
 731 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
 732 {
 733         struct unw_reg_info *reg;
 734
 735         for (reg = *regp; reg <= lim; ++reg) {
 736                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 737                         reg->when = t;
 738                         *regp = reg + 1;
 739                         return;
 740                 }
 741         }
 742         UNW_DPRINT(0, "unwind.%s: excess spill!\n",  __func__);
 743 }
 744
 745 static inline void
 746 finish_prologue (struct unw_state_record *sr)
 747 {
 748         struct unw_reg_info *reg;
 749         unsigned long off;
 750         int i;
 751
 752         /*
 753          * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
 754          * for Using Unwind Descriptors", rule 3):
 755          */
 756         for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
 757                 reg = sr->curr.reg + unw.save_order[i];
 758                 if (reg->where == UNW_WHERE_GR_SAVE) {
 759                         reg->where = UNW_WHERE_GR;
 760                         reg->val = sr->gr_save_loc++;
 761                 }
 762         }
 763
 764         /*
 765          * Next, compute when the fp, general, and branch registers get
 766          * saved.  This must come before alloc_spill_area() because
 767          * we need to know which registers are spilled to their home
 768          * locations.
 769          */
 770         if (sr->imask) {
 771                 unsigned char kind, mask = 0, *cp = sr->imask;
 772                 int t;
 773                 static const unsigned char limit[3] = {
 774                         UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
 775                 };
 776                 struct unw_reg_info *(regs[3]);
 777
 778                 regs[0] = sr->curr.reg + UNW_REG_F2;
 779                 regs[1] = sr->curr.reg + UNW_REG_R4;
 780                 regs[2] = sr->curr.reg + UNW_REG_B1;
 781
 782                 for (t = 0; t < sr->region_len; ++t) {
 783                         if ((t & 3) == 0)
 784                                 mask = *cp++;
 785                         kind = (mask >> 2*(3-(t & 3))) & 3;
 786                         if (kind > 0)
 787                                 spill_next_when(&regs[kind - 1], sr->curr.reg + limit[kind - 1],
 788                                                 sr->region_start + t);
 789                 }
 790         }
 791         /*
 792          * Next, lay out the memory stack spill area:
 793          */
 794         if (sr->any_spills) {
 795                 off = sr->spill_offset;
 796                 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
 797                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
 798                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
 799         }
 800 }
 801
 802 /*
 803  * Region header descriptors.
 804  */
 805
 806 static void
 807 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
 808                struct unw_state_record *sr)
 809 {
 810         int i, region_start;
 811
 812         if (!(sr->in_body || sr->first_region))
 813                 finish_prologue(sr);
 814         sr->first_region = 0;
 815
 816         /* check if we're done: */
 817         if (sr->when_target < sr->region_start + sr->region_len) {
 818                 sr->done = 1;
 819                 return;
 820         }
 821
 822         region_start = sr->region_start + sr->region_len;
 823
 824         for (i = 0; i < sr->epilogue_count; ++i)
 825                 pop(sr);
 826         sr->epilogue_count = 0;
 827         sr->epilogue_start = UNW_WHEN_NEVER;
 828
 829         sr->region_start = region_start;
 830         sr->region_len = rlen;
 831         sr->in_body = body;
 832
 833         if (!body) {
 834                 push(sr);
 835
 836                 for (i = 0; i < 4; ++i) {
 837                         if (mask & 0x8)
 838                                 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
 839                                         sr->region_start + sr->region_len - 1, grsave++);
 840                         mask <<= 1;
 841                 }
 842                 sr->gr_save_loc = grsave;
 843                 sr->any_spills = 0;
 844                 sr->imask = NULL;
 845                 sr->spill_offset = 0x10;        /* default to psp+16 */
 846         }
 847 }
 848
 849 /*
 850  * Prologue descriptors.
 851  */
 852
 853 static inline void
 854 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
 855 {
 856         if (abi == 3 && context == 'i') {
 857                 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
 858                 UNW_DPRINT(3, "unwind.%s: interrupt frame\n",  __func__);
 859         }
 860         else
 861                 UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
 862                                 __func__, abi, context);
 863 }
 864
 865 static inline void
 866 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
 867 {
 868         int i;
 869
 870         for (i = 0; i < 5; ++i) {
 871                 if (brmask & 1)
 872                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
 873                                 sr->region_start + sr->region_len - 1, gr++);
 874                 brmask >>= 1;
 875         }
 876 }
 877
 878 static inline void
 879 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
 880 {
 881         int i;
 882
 883         for (i = 0; i < 5; ++i) {
 884                 if (brmask & 1) {
 885                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
 886                                 sr->region_start + sr->region_len - 1, 0);
 887                         sr->any_spills = 1;
 888                 }
 889                 brmask >>= 1;
 890         }
 891 }
 892
 893 static inline void
 894 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
 895 {
 896         int i;
 897
 898         for (i = 0; i < 4; ++i) {
 899                 if ((grmask & 1) != 0) {
 900                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 901                                 sr->region_start + sr->region_len - 1, 0);
 902                         sr->any_spills = 1;
 903                 }
 904                 grmask >>= 1;
 905         }
 906         for (i = 0; i < 20; ++i) {
 907                 if ((frmask & 1) != 0) {
 908                         int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
 909                         set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
 910                                 sr->region_start + sr->region_len - 1, 0);
 911                         sr->any_spills = 1;
 912                 }
 913                 frmask >>= 1;
 914         }
 915 }
 916
 917 static inline void
 918 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
 919 {
 920         int i;
 921
 922         for (i = 0; i < 4; ++i) {
 923                 if ((frmask & 1) != 0) {
 924                         set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
 925                                 sr->region_start + sr->region_len - 1, 0);
 926                         sr->any_spills = 1;
 927                 }
 928                 frmask >>= 1;
 929         }
 930 }
 931
 932 static inline void
 933 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
 934 {
 935         int i;
 936
 937         for (i = 0; i < 4; ++i) {
 938                 if ((grmask & 1) != 0)
 939                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
 940                                 sr->region_start + sr->region_len - 1, gr++);
 941                 grmask >>= 1;
 942         }
 943 }
 944
 945 static inline void
 946 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
 947 {
 948         int i;
 949
 950         for (i = 0; i < 4; ++i) {
 951                 if ((grmask & 1) != 0) {
 952                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 953                                 sr->region_start + sr->region_len - 1, 0);
 954                         sr->any_spills = 1;
 955                 }
 956                 grmask >>= 1;
 957         }
 958 }
 959
 960 static inline void
 961 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
 962 {
 963         set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
 964                 sr->region_start + min_t(int, t, sr->region_len - 1), 16*size);
 965 }
 966
 967 static inline void
 968 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
 969 {
 970         sr->curr.reg[UNW_REG_PSP].when = sr->region_start + min_t(int, t, sr->region_len - 1);
 971 }
 972
 973 static inline void
 974 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
 975 {
 976         set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
 977 }
 978
 979 static inline void
 980 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
 981 {
 982         set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
 983                 0x10 - 4*pspoff);
 984 }
 985
 986 static inline void
 987 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
 988 {
 989         set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
 990                 4*spoff);
 991 }
 992
 993 static inline void
 994 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
 995 {
 996         sr->return_link_reg = dst;
 997 }
 998
 999 static inline void
1000 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
1001 {
1002         struct unw_reg_info *reg = sr->curr.reg + regnum;
1003
1004         if (reg->where == UNW_WHERE_NONE)
1005                 reg->where = UNW_WHERE_GR_SAVE;
1006         reg->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1007 }
1008
1009 static inline void
1010 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
1011 {
1012         sr->spill_offset = 0x10 - 4*pspoff;
1013 }
1014
1015 static inline unsigned char *
1016 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
1017 {
1018         sr->imask = imaskp;
1019         return imaskp + (2*sr->region_len + 7)/8;
1020 }
1021
1022 /*
1023  * Body descriptors.
1024  */
1025 static inline void
1026 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
1027 {
1028         sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
1029         sr->epilogue_count = ecount + 1;
1030 }
1031
1032 static inline void
1033 desc_copy_state (unw_word label, struct unw_state_record *sr)
1034 {
1035         struct unw_labeled_state *ls;
1036
1037         for (ls = sr->labeled_states; ls; ls = ls->next) {
1038                 if (ls->label == label) {
1039                         free_state_stack(&sr->curr);
1040                         memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
1041                         sr->curr.next = dup_state_stack(ls->saved_state.next);
1042                         return;
1043                 }
1044         }
1045         printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
1046 }
1047
1048 static inline void
1049 desc_label_state (unw_word label, struct unw_state_record *sr)
1050 {
1051         struct unw_labeled_state *ls;
1052
1053         ls = alloc_labeled_state();
1054         if (!ls) {
1055                 printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
1056                 return;
1057         }
1058         ls->label = label;
1059         memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
1060         ls->saved_state.next = dup_state_stack(sr->curr.next);
1061
1062         /* insert into list of labeled states: */
1063         ls->next = sr->labeled_states;
1064         sr->labeled_states = ls;
1065 }
1066
1067 /*
1068  * General descriptors.
1069  */
1070
1071 static inline int
1072 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1073 {
1074         if (sr->when_target <= sr->region_start + min_t(int, t, sr->region_len - 1))
1075                 return 0;
1076         if (qp > 0) {
1077                 if ((sr->pr_val & (1UL << qp)) == 0)
1078                         return 0;
1079                 sr->pr_mask |= (1UL << qp);
1080         }
1081         return 1;
1082 }
1083
1084 static inline void
1085 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1086 {
1087         struct unw_reg_info *r;
1088
1089         if (!desc_is_active(qp, t, sr))
1090                 return;
1091
1092         r = sr->curr.reg + decode_abreg(abreg, 0);
1093         r->where = UNW_WHERE_NONE;
1094         r->when = UNW_WHEN_NEVER;
1095         r->val = 0;
1096 }
1097
1098 static inline void
1099 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1100                      unsigned char ytreg, struct unw_state_record *sr)
1101 {
1102         enum unw_where where = UNW_WHERE_GR;
1103         struct unw_reg_info *r;
1104
1105         if (!desc_is_active(qp, t, sr))
1106                 return;
1107
1108         if (x)
1109                 where = UNW_WHERE_BR;
1110         else if (ytreg & 0x80)
1111                 where = UNW_WHERE_FR;
1112
1113         r = sr->curr.reg + decode_abreg(abreg, 0);
1114         r->where = where;
1115         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1116         r->val = (ytreg & 0x7f);
1117 }
1118
1119 static inline void
1120 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1121                      struct unw_state_record *sr)
1122 {
1123         struct unw_reg_info *r;
1124
1125         if (!desc_is_active(qp, t, sr))
1126                 return;
1127
1128         r = sr->curr.reg + decode_abreg(abreg, 1);
1129         r->where = UNW_WHERE_PSPREL;
1130         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1131         r->val = 0x10 - 4*pspoff;
1132 }
1133
1134 static inline void
1135 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1136                        struct unw_state_record *sr)
1137 {
1138         struct unw_reg_info *r;
1139
1140         if (!desc_is_active(qp, t, sr))
1141                 return;
1142
1143         r = sr->curr.reg + decode_abreg(abreg, 1);
1144         r->where = UNW_WHERE_SPREL;
1145         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1146         r->val = 4*spoff;
1147 }
1148
1149 #define UNW_DEC_BAD_CODE(code)                  printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1150                                                        code);
1151
1152 /*
1153  * region headers:
1154  */
1155 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg)     desc_prologue(0,r,m,gr,arg)
1156 #define UNW_DEC_PROLOGUE(fmt,b,r,arg)           desc_prologue(b,r,0,32,arg)
1157 /*
1158  * prologue descriptors:
1159  */
1160 #define UNW_DEC_ABI(fmt,a,c,arg)                desc_abi(a,c,arg)
1161 #define UNW_DEC_BR_GR(fmt,b,g,arg)              desc_br_gr(b,g,arg)
1162 #define UNW_DEC_BR_MEM(fmt,b,arg)               desc_br_mem(b,arg)
1163 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg)           desc_frgr_mem(g,f,arg)
1164 #define UNW_DEC_FR_MEM(fmt,f,arg)               desc_fr_mem(f,arg)
1165 #define UNW_DEC_GR_GR(fmt,m,g,arg)              desc_gr_gr(m,g,arg)
1166 #define UNW_DEC_GR_MEM(fmt,m,arg)               desc_gr_mem(m,arg)
1167 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg)        desc_mem_stack_f(t,s,arg)
1168 #define UNW_DEC_MEM_STACK_V(fmt,t,arg)          desc_mem_stack_v(t,arg)
1169 #define UNW_DEC_REG_GR(fmt,r,d,arg)             desc_reg_gr(r,d,arg)
1170 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg)         desc_reg_psprel(r,o,arg)
1171 #define UNW_DEC_REG_SPREL(fmt,r,o,arg)          desc_reg_sprel(r,o,arg)
1172 #define UNW_DEC_REG_WHEN(fmt,r,t,arg)           desc_reg_when(r,t,arg)
1173 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg)      desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1174 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg)     desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1175 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg)           desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1176 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg)       desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1177 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg)        desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1178 #define UNW_DEC_RP_BR(fmt,d,arg)                desc_rp_br(d,arg)
1179 #define UNW_DEC_SPILL_BASE(fmt,o,arg)           desc_spill_base(o,arg)
1180 #define UNW_DEC_SPILL_MASK(fmt,m,arg)           (m = desc_spill_mask(m,arg))
1181 /*
1182  * body descriptors:
1183  */
1184 #define UNW_DEC_EPILOGUE(fmt,t,c,arg)           desc_epilogue(t,c,arg)
1185 #define UNW_DEC_COPY_STATE(fmt,l,arg)           desc_copy_state(l,arg)
1186 #define UNW_DEC_LABEL_STATE(fmt,l,arg)          desc_label_state(l,arg)
1187 /*
1188  * general unwind descriptors:
1189  */
1190 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg)    desc_spill_reg_p(p,t,a,x,y,arg)
1191 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg)        desc_spill_reg_p(0,t,a,x,y,arg)
1192 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg)   desc_spill_psprel_p(p,t,a,o,arg)
1193 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg)       desc_spill_psprel_p(0,t,a,o,arg)
1194 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg)    desc_spill_sprel_p(p,t,a,o,arg)
1195 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg)        desc_spill_sprel_p(0,t,a,o,arg)
1196 #define UNW_DEC_RESTORE_P(f,p,t,a,arg)          desc_restore_p(p,t,a,arg)
1197 #define UNW_DEC_RESTORE(f,t,a,arg)              desc_restore_p(0,t,a,arg)
1198
1199 #include "unwind_decoder.c"
1200
1201 \f
1202 /* Unwind scripts. */
1203
1204 static inline unw_hash_index_t
1205 hash (unsigned long ip)
1206 {
1207         /* magic number = ((sqrt(5)-1)/2)*2^64 */
1208         static const unsigned long hashmagic = 0x9e3779b97f4a7c16UL;
1209
1210         return (ip >> 4) * hashmagic >> (64 - UNW_LOG_HASH_SIZE);
1211 }
1212
1213 static inline long
1214 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1215 {
1216         read_lock(&script->lock);
1217         if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1218                 /* keep the read lock... */
1219                 return 1;
1220         read_unlock(&script->lock);
1221         return 0;
1222 }
1223
1224 static inline struct unw_script *
1225 script_lookup (struct unw_frame_info *info)
1226 {
1227         struct unw_script *script = unw.cache + info->hint;
1228         unsigned short index;
1229         unsigned long ip, pr;
1230
1231         if (UNW_DEBUG_ON(0))
1232                 return NULL;    /* Always regenerate scripts in debug mode */
1233
1234         STAT(++unw.stat.cache.lookups);
1235
1236         ip = info->ip;
1237         pr = info->pr;
1238
1239         if (cache_match(script, ip, pr)) {
1240                 STAT(++unw.stat.cache.hinted_hits);
1241                 return script;
1242         }
1243
1244         index = unw.hash[hash(ip)];
1245         if (index >= UNW_CACHE_SIZE)
1246                 return NULL;
1247
1248         script = unw.cache + index;
1249         while (1) {
1250                 if (cache_match(script, ip, pr)) {
1251                         /* update hint; no locking required as single-word writes are atomic */
1252                         STAT(++unw.stat.cache.normal_hits);
1253                         unw.cache[info->prev_script].hint = script - unw.cache;
1254                         return script;
1255                 }
1256                 if (script->coll_chain >= UNW_HASH_SIZE)
1257                         return NULL;
1258                 script = unw.cache + script->coll_chain;
1259                 STAT(++unw.stat.cache.collision_chain_traversals);
1260         }
1261 }
1262
1263 /*
1264  * On returning, a write lock for the SCRIPT is still being held.
1265  */
1266 static inline struct unw_script *
1267 script_new (unsigned long ip)
1268 {
1269         struct unw_script *script, *prev, *tmp;
1270         unw_hash_index_t index;
1271         unsigned short head;
1272
1273         STAT(++unw.stat.script.news);
1274
1275         /*
1276          * Can't (easily) use cmpxchg() here because of ABA problem
1277          * that is intrinsic in cmpxchg()...
1278          */
1279         head = unw.lru_head;
1280         script = unw.cache + head;
1281         unw.lru_head = script->lru_chain;
1282
1283         /*
1284          * We'd deadlock here if we interrupted a thread that is holding a read lock on
1285          * script->lock.  Thus, if the write_trylock() fails, we simply bail out.  The
1286          * alternative would be to disable interrupts whenever we hold a read-lock, but
1287          * that seems silly.
1288          */
1289         if (!write_trylock(&script->lock))
1290                 return NULL;
1291
1292         /* re-insert script at the tail of the LRU chain: */
1293         unw.cache[unw.lru_tail].lru_chain = head;
1294         unw.lru_tail = head;
1295
1296         /* remove the old script from the hash table (if it's there): */
1297         if (script->ip) {
1298                 index = hash(script->ip);
1299                 tmp = unw.cache + unw.hash[index];
1300                 prev = NULL;
1301                 while (1) {
1302                         if (tmp == script) {
1303                                 if (prev)
1304                                         prev->coll_chain = tmp->coll_chain;
1305                                 else
1306                                         unw.hash[index] = tmp->coll_chain;
1307                                 break;
1308                         } else
1309                                 prev = tmp;
1310                         if (tmp->coll_chain >= UNW_CACHE_SIZE)
1311                         /* old script wasn't in the hash-table */
1312                                 break;
1313                         tmp = unw.cache + tmp->coll_chain;
1314                 }
1315         }
1316
1317         /* enter new script in the hash table */
1318         index = hash(ip);
1319         script->coll_chain = unw.hash[index];
1320         unw.hash[index] = script - unw.cache;
1321
1322         script->ip = ip;        /* set new IP while we're holding the locks */
1323
1324         STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1325
1326         script->flags = 0;
1327         script->hint = 0;
1328         script->count = 0;
1329         return script;
1330 }
1331
1332 static void
1333 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1334 {
1335         script->pr_mask = sr->pr_mask;
1336         script->pr_val = sr->pr_val;
1337         /*
1338          * We could down-grade our write-lock on script->lock here but
1339          * the rwlock API doesn't offer atomic lock downgrading, so
1340          * we'll just keep the write-lock and release it later when
1341          * we're done using the script.
1342          */
1343 }
1344
1345 static inline void
1346 script_emit (struct unw_script *script, struct unw_insn insn)
1347 {
1348         if (script->count >= UNW_MAX_SCRIPT_LEN) {
1349                 UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1350                         __func__, UNW_MAX_SCRIPT_LEN);
1351                 return;
1352         }
1353         script->insn[script->count++] = insn;
1354 }
1355
1356 static inline void
1357 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1358 {
1359         struct unw_reg_info *r = sr->curr.reg + i;
1360         enum unw_insn_opcode opc;
1361         struct unw_insn insn;
1362         unsigned long val = 0;
1363
1364         switch (r->where) {
1365               case UNW_WHERE_GR:
1366                 if (r->val >= 32) {
1367                         /* register got spilled to a stacked register */
1368                         opc = UNW_INSN_SETNAT_TYPE;
1369                         val = UNW_NAT_REGSTK;
1370                 } else
1371                         /* register got spilled to a scratch register */
1372                         opc = UNW_INSN_SETNAT_MEMSTK;
1373                 break;
1374
1375               case UNW_WHERE_FR:
1376                 opc = UNW_INSN_SETNAT_TYPE;
1377                 val = UNW_NAT_VAL;
1378                 break;
1379
1380               case UNW_WHERE_BR:
1381                 opc = UNW_INSN_SETNAT_TYPE;
1382                 val = UNW_NAT_NONE;
1383                 break;
1384
1385               case UNW_WHERE_PSPREL:
1386               case UNW_WHERE_SPREL:
1387                 opc = UNW_INSN_SETNAT_MEMSTK;
1388                 break;
1389
1390               default:
1391                 UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1392                            __func__, r->where);
1393                 return;
1394         }
1395         insn.opc = opc;
1396         insn.dst = unw.preg_index[i];
1397         insn.val = val;
1398         script_emit(script, insn);
1399 }
1400
1401 static void
1402 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1403 {
1404         struct unw_reg_info *r = sr->curr.reg + i;
1405         enum unw_insn_opcode opc;
1406         unsigned long val, rval;
1407         struct unw_insn insn;
1408         long need_nat_info;
1409
1410         if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1411                 return;
1412
1413         opc = UNW_INSN_MOVE;
1414         val = rval = r->val;
1415         need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1416
1417         switch (r->where) {
1418               case UNW_WHERE_GR:
1419                 if (rval >= 32) {
1420                         opc = UNW_INSN_MOVE_STACKED;
1421                         val = rval - 32;
1422                 } else if (rval >= 4 && rval <= 7) {
1423                         if (need_nat_info) {
1424                                 opc = UNW_INSN_MOVE2;
1425                                 need_nat_info = 0;
1426                         }
1427                         val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1428                 } else if (rval == 0) {
1429                         opc = UNW_INSN_MOVE_CONST;
1430                         val = 0;
1431                 } else {
1432                         /* register got spilled to a scratch register */
1433                         opc = UNW_INSN_MOVE_SCRATCH;
1434                         val = pt_regs_off(rval);
1435                 }
1436                 break;
1437
1438               case UNW_WHERE_FR:
1439                 if (rval <= 5)
1440                         val = unw.preg_index[UNW_REG_F2  + (rval -  2)];
1441                 else if (rval >= 16 && rval <= 31)
1442                         val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1443                 else {
1444                         opc = UNW_INSN_MOVE_SCRATCH;
1445                         if (rval <= 11)
1446                                 val = offsetof(struct pt_regs, f6) + 16*(rval - 6);
1447                         else
1448                                 UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1449                                            __func__, rval);
1450                 }
1451                 break;
1452
1453               case UNW_WHERE_BR:
1454                 if (rval >= 1 && rval <= 5)
1455                         val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1456                 else {
1457                         opc = UNW_INSN_MOVE_SCRATCH;
1458                         if (rval == 0)
1459                                 val = offsetof(struct pt_regs, b0);
1460                         else if (rval == 6)
1461                                 val = offsetof(struct pt_regs, b6);
1462                         else
1463                                 val = offsetof(struct pt_regs, b7);
1464                 }
1465                 break;
1466
1467               case UNW_WHERE_SPREL:
1468                 opc = UNW_INSN_ADD_SP;
1469                 break;
1470
1471               case UNW_WHERE_PSPREL:
1472                 opc = UNW_INSN_ADD_PSP;
1473                 break;
1474
1475               default:
1476                 UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1477                            __func__, i, r->where);
1478                 break;
1479         }
1480         insn.opc = opc;
1481         insn.dst = unw.preg_index[i];
1482         insn.val = val;
1483         script_emit(script, insn);
1484         if (need_nat_info)
1485                 emit_nat_info(sr, i, script);
1486
1487         if (i == UNW_REG_PSP) {
1488                 /*
1489                  * info->psp must contain the _value_ of the previous
1490                  * sp, not it's save location.  We get this by
1491                  * dereferencing the value we just stored in
1492                  * info->psp:
1493                  */
1494                 insn.opc = UNW_INSN_LOAD;
1495                 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1496                 script_emit(script, insn);
1497         }
1498 }
1499
1500 static inline const struct unw_table_entry *
1501 lookup (struct unw_table *table, unsigned long rel_ip)
1502 {
1503         const struct unw_table_entry *e = NULL;
1504         unsigned long lo, hi, mid;
1505
1506         /* do a binary search for right entry: */
1507         for (lo = 0, hi = table->length; lo < hi; ) {
1508                 mid = (lo + hi) / 2;
1509                 e = &table->array[mid];
1510                 if (rel_ip < e->start_offset)
1511                         hi = mid;
1512                 else if (rel_ip >= e->end_offset)
1513                         lo = mid + 1;
1514                 else
1515                         break;
1516         }
1517         if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1518                 return NULL;
1519         return e;
1520 }
1521
1522 /*
1523  * Build an unwind script that unwinds from state OLD_STATE to the
1524  * entrypoint of the function that called OLD_STATE.
1525  */
1526 static inline struct unw_script *
1527 build_script (struct unw_frame_info *info)
1528 {
1529         const struct unw_table_entry *e = NULL;
1530         struct unw_script *script = NULL;
1531         struct unw_labeled_state *ls, *next;
1532         unsigned long ip = info->ip;
1533         struct unw_state_record sr;
1534         struct unw_table *table, *prev;
1535         struct unw_reg_info *r;
1536         struct unw_insn insn;
1537         u8 *dp, *desc_end;
1538         u64 hdr;
1539         int i;
1540         STAT(unsigned long start, parse_start;)
1541
1542         STAT(++unw.stat.script.builds; start = ia64_get_itc());
1543
1544         /* build state record */
1545         memset(&sr, 0, sizeof(sr));
1546         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1547                 r->when = UNW_WHEN_NEVER;
1548         sr.pr_val = info->pr;
1549
1550         UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __func__, ip);
1551         script = script_new(ip);
1552         if (!script) {
1553                 UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n",  __func__);
1554                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1555                 return NULL;
1556         }
1557         unw.cache[info->prev_script].hint = script - unw.cache;
1558
1559         /* search the kernels and the modules' unwind tables for IP: */
1560
1561         STAT(parse_start = ia64_get_itc());
1562
1563         prev = NULL;
1564         for (table = unw.tables; table; table = table->next) {
1565                 if (ip >= table->start && ip < table->end) {
1566                         /*
1567                          * Leave the kernel unwind table at the very front,
1568                          * lest moving it breaks some assumption elsewhere.
1569                          * Otherwise, move the matching table to the second
1570                          * position in the list so that traversals can benefit
1571                          * from commonality in backtrace paths.
1572                          */
1573                         if (prev && prev != unw.tables) {
1574                                 /* unw is safe - we're already spinlocked */
1575                                 prev->next = table->next;
1576                                 table->next = unw.tables->next;
1577                                 unw.tables->next = table;
1578                         }
1579                         e = lookup(table, ip - table->segment_base);
1580                         break;
1581                 }
1582                 prev = table;
1583         }
1584         if (!e) {
1585                 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs)  */
1586                 UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1587                         __func__, ip, unw.cache[info->prev_script].ip);
1588                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1589                 sr.curr.reg[UNW_REG_RP].when = -1;
1590                 sr.curr.reg[UNW_REG_RP].val = 0;
1591                 compile_reg(&sr, UNW_REG_RP, script);
1592                 script_finalize(script, &sr);
1593                 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1594                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1595                 return script;
1596         }
1597
1598         sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1599                           + (ip & 0xfUL));
1600         hdr = *(u64 *) (table->segment_base + e->info_offset);
1601         dp =   (u8 *)  (table->segment_base + e->info_offset + 8);
1602         desc_end = dp + 8*UNW_LENGTH(hdr);
1603
1604         while (!sr.done && dp < desc_end)
1605                 dp = unw_decode(dp, sr.in_body, &sr);
1606
1607         if (sr.when_target > sr.epilogue_start) {
1608                 /*
1609                  * sp has been restored and all values on the memory stack below
1610                  * psp also have been restored.
1611                  */
1612                 sr.curr.reg[UNW_REG_PSP].val = 0;
1613                 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1614                 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1615                 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1616                         if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1617                             || r->where == UNW_WHERE_SPREL)
1618                         {
1619                                 r->val = 0;
1620                                 r->where = UNW_WHERE_NONE;
1621                                 r->when = UNW_WHEN_NEVER;
1622                         }
1623         }
1624
1625         script->flags = sr.flags;
1626
1627         /*
1628          * If RP did't get saved, generate entry for the return link
1629          * register.
1630          */
1631         if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1632                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1633                 sr.curr.reg[UNW_REG_RP].when = -1;
1634                 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1635                 UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1636                            __func__, ip, sr.curr.reg[UNW_REG_RP].where,
1637                            sr.curr.reg[UNW_REG_RP].val);
1638         }
1639
1640 #ifdef UNW_DEBUG
1641         UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1642                 __func__, table->segment_base + e->start_offset, sr.when_target);
1643         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1644                 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1645                         UNW_DPRINT(1, "  %s <- ", unw.preg_name[r - sr.curr.reg]);
1646                         switch (r->where) {
1647                               case UNW_WHERE_GR:     UNW_DPRINT(1, "r%lu", r->val); break;
1648                               case UNW_WHERE_FR:     UNW_DPRINT(1, "f%lu", r->val); break;
1649                               case UNW_WHERE_BR:     UNW_DPRINT(1, "b%lu", r->val); break;
1650                               case UNW_WHERE_SPREL:  UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1651                               case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1652                               case UNW_WHERE_NONE:
1653                                 UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1654                                 break;
1655
1656                               default:
1657                                 UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1658                                 break;
1659                         }
1660                         UNW_DPRINT(1, "\t\t%d\n", r->when);
1661                 }
1662         }
1663 #endif
1664
1665         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1666
1667         /* translate state record into unwinder instructions: */
1668
1669         /*
1670          * First, set psp if we're dealing with a fixed-size frame;
1671          * subsequent instructions may depend on this value.
1672          */
1673         if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1674             && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1675             && sr.curr.reg[UNW_REG_PSP].val != 0) {
1676                 /* new psp is sp plus frame size */
1677                 insn.opc = UNW_INSN_ADD;
1678                 insn.dst = offsetof(struct unw_frame_info, psp)/8;
1679                 insn.val = sr.curr.reg[UNW_REG_PSP].val;        /* frame size */
1680                 script_emit(script, insn);
1681         }
1682
1683         /* determine where the primary UNaT is: */
1684         if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1685                 i = UNW_REG_PRI_UNAT_MEM;
1686         else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1687                 i = UNW_REG_PRI_UNAT_GR;
1688         else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1689                 i = UNW_REG_PRI_UNAT_MEM;
1690         else
1691                 i = UNW_REG_PRI_UNAT_GR;
1692
1693         compile_reg(&sr, i, script);
1694
1695         for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1696                 compile_reg(&sr, i, script);
1697
1698         /* free labeled register states & stack: */
1699
1700         STAT(parse_start = ia64_get_itc());
1701         for (ls = sr.labeled_states; ls; ls = next) {
1702                 next = ls->next;
1703                 free_state_stack(&ls->saved_state);
1704                 free_labeled_state(ls);
1705         }
1706         free_state_stack(&sr.curr);
1707         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1708
1709         script_finalize(script, &sr);
1710         STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1711         return script;
1712 }
1713
1714 /*
1715  * Apply the unwinding actions represented by OPS and update SR to
1716  * reflect the state that existed upon entry to the function that this
1717  * unwinder represents.
1718  */
1719 static inline void
1720 run_script (struct unw_script *script, struct unw_frame_info *state)
1721 {
1722         struct unw_insn *ip, *limit, next_insn;
1723         unsigned long opc, dst, val, off;
1724         unsigned long *s = (unsigned long *) state;
1725         STAT(unsigned long start;)
1726
1727         STAT(++unw.stat.script.runs; start = ia64_get_itc());
1728         state->flags = script->flags;
1729         ip = script->insn;
1730         limit = script->insn + script->count;
1731         next_insn = *ip;
1732
1733         while (ip++ < limit) {
1734                 opc = next_insn.opc;
1735                 dst = next_insn.dst;
1736                 val = next_insn.val;
1737                 next_insn = *ip;
1738
1739           redo:
1740                 switch (opc) {
1741                       case UNW_INSN_ADD:
1742                         s[dst] += val;
1743                         break;
1744
1745                       case UNW_INSN_MOVE2:
1746                         if (!s[val])
1747                                 goto lazy_init;
1748                         s[dst+1] = s[val+1];
1749                         s[dst] = s[val];
1750                         break;
1751
1752                       case UNW_INSN_MOVE:
1753                         if (!s[val])
1754                                 goto lazy_init;
1755                         s[dst] = s[val];
1756                         break;
1757
1758                       case UNW_INSN_MOVE_SCRATCH:
1759                         if (state->pt) {
1760                                 s[dst] = (unsigned long) get_scratch_regs(state) + val;
1761                         } else {
1762                                 s[dst] = 0;
1763                                 UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1764                                            __func__, dst, val);
1765                         }
1766                         break;
1767
1768                       case UNW_INSN_MOVE_CONST:
1769                         if (val == 0)
1770                                 s[dst] = (unsigned long) &unw.r0;
1771                         else {
1772                                 s[dst] = 0;
1773                                 UNW_DPRINT(0, "unwind.%s: UNW_INSN_MOVE_CONST bad val=%ld\n",
1774                                            __func__, val);
1775                         }
1776                         break;
1777
1778
1779                       case UNW_INSN_MOVE_STACKED:
1780                         s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1781                                                                     val);
1782                         break;
1783
1784                       case UNW_INSN_ADD_PSP:
1785                         s[dst] = state->psp + val;
1786                         break;
1787
1788                       case UNW_INSN_ADD_SP:
1789                         s[dst] = state->sp + val;
1790                         break;
1791
1792                       case UNW_INSN_SETNAT_MEMSTK:
1793                         if (!state->pri_unat_loc)
1794                                 state->pri_unat_loc = &state->sw->caller_unat;
1795                         /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1796                         s[dst+1] = ((unsigned long) state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1797                         break;
1798
1799                       case UNW_INSN_SETNAT_TYPE:
1800                         s[dst+1] = val;
1801                         break;
1802
1803                       case UNW_INSN_LOAD:
1804 #ifdef UNW_DEBUG
1805                         if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1806                             || s[val] < TASK_SIZE)
1807                         {
1808                                 UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1809                                            __func__, s[val]);
1810                                 break;
1811                         }
1812 #endif
1813                         s[dst] = *(unsigned long *) s[val];
1814                         break;
1815                 }
1816         }
1817         STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1818         return;
1819
1820   lazy_init:
1821         off = unw.sw_off[val];
1822         s[val] = (unsigned long) state->sw + off;
1823         if (off >= offsetof(struct switch_stack, r4) && off <= offsetof(struct switch_stack, r7))
1824                 /*
1825                  * We're initializing a general register: init NaT info, too.  Note that
1826                  * the offset is a multiple of 8 which gives us the 3 bits needed for
1827                  * the type field.
1828                  */
1829                 s[val+1] = (offsetof(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1830         goto redo;
1831 }
1832
1833 static int
1834 find_save_locs (struct unw_frame_info *info)
1835 {
1836         int have_write_lock = 0;
1837         struct unw_script *scr;
1838         unsigned long flags = 0;
1839
1840         if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1841                 /* don't let obviously bad addresses pollute the cache */
1842                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1843                 UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __func__, info->ip);
1844                 info->rp_loc = NULL;
1845                 return -1;
1846         }
1847
1848         scr = script_lookup(info);
1849         if (!scr) {
1850                 spin_lock_irqsave(&unw.lock, flags);
1851                 scr = build_script(info);
1852                 if (!scr) {
1853                         spin_unlock_irqrestore(&unw.lock, flags);
1854                         UNW_DPRINT(0,
1855                                    "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1856                                    __func__, info->ip);
1857                         return -1;
1858                 }
1859                 have_write_lock = 1;
1860         }
1861         info->hint = scr->hint;
1862         info->prev_script = scr - unw.cache;
1863
1864         run_script(scr, info);
1865
1866         if (have_write_lock) {
1867                 write_unlock(&scr->lock);
1868                 spin_unlock_irqrestore(&unw.lock, flags);
1869         } else
1870                 read_unlock(&scr->lock);
1871         return 0;
1872 }
1873
1874 static int
1875 unw_valid(const struct unw_frame_info *info, unsigned long* p)
1876 {
1877         unsigned long loc = (unsigned long)p;
1878         return (loc >= info->regstk.limit && loc < info->regstk.top) ||
1879                (loc >= info->memstk.top && loc < info->memstk.limit);
1880 }
1881
1882 int
1883 unw_unwind (struct unw_frame_info *info)
1884 {
1885         unsigned long prev_ip, prev_sp, prev_bsp;
1886         unsigned long ip, pr, num_regs;
1887         STAT(unsigned long start, flags;)
1888         int retval;
1889
1890         STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1891
1892         prev_ip = info->ip;
1893         prev_sp = info->sp;
1894         prev_bsp = info->bsp;
1895
1896         /* validate the return IP pointer */
1897         if (!unw_valid(info, info->rp_loc)) {
1898                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1899                 UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1900                            __func__, info->ip);
1901                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1902                 return -1;
1903         }
1904         /* restore the ip */
1905         ip = info->ip = *info->rp_loc;
1906         if (ip < GATE_ADDR) {
1907                 UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __func__, ip);
1908                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1909                 return -1;
1910         }
1911
1912         /* validate the previous stack frame pointer */
1913         if (!unw_valid(info, info->pfs_loc)) {
1914                 UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __func__);
1915                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1916                 return -1;
1917         }
1918         /* restore the cfm: */
1919         info->cfm_loc = info->pfs_loc;
1920
1921         /* restore the bsp: */
1922         pr = info->pr;
1923         num_regs = 0;
1924         if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1925                 info->pt = info->sp + 16;
1926                 if ((pr & (1UL << PRED_NON_SYSCALL)) != 0)
1927                         num_regs = *info->cfm_loc & 0x7f;               /* size of frame */
1928                 info->pfs_loc =
1929                         (unsigned long *) (info->pt + offsetof(struct pt_regs, ar_pfs));
1930                 UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __func__, info->pt);
1931         } else
1932                 num_regs = (*info->cfm_loc >> 7) & 0x7f;        /* size of locals */
1933         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1934         if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1935                 UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1936                         __func__, info->bsp, info->regstk.limit, info->regstk.top);
1937                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1938                 return -1;
1939         }
1940
1941         /* restore the sp: */
1942         info->sp = info->psp;
1943         if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1944                 UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1945                         __func__, info->sp, info->memstk.top, info->memstk.limit);
1946                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1947                 return -1;
1948         }
1949
1950         if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1951                 UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1952                            __func__, ip);
1953                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1954                 return -1;
1955         }
1956
1957         /* as we unwind, the saved ar.unat becomes the primary unat: */
1958         info->pri_unat_loc = info->unat_loc;
1959
1960         /* finally, restore the predicates: */
1961         unw_get_pr(info, &info->pr);
1962
1963         retval = find_save_locs(info);
1964         STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1965         return retval;
1966 }
1967 EXPORT_SYMBOL(unw_unwind);
1968
1969 int
1970 unw_unwind_to_user (struct unw_frame_info *info)
1971 {
1972         unsigned long ip, sp, pr = info->pr;
1973
1974         do {
1975                 unw_get_sp(info, &sp);
1976                 if ((long)((unsigned long)info->task + IA64_STK_OFFSET - sp)
1977                     < IA64_PT_REGS_SIZE) {
1978                         UNW_DPRINT(0, "unwind.%s: ran off the top of the kernel stack\n",
1979                                    __func__);
1980                         break;
1981                 }
1982                 if (unw_is_intr_frame(info) &&
1983                     (pr & (1UL << PRED_USER_STACK)))
1984                         return 0;
1985                 if (unw_get_pr (info, &pr) < 0) {
1986                         unw_get_rp(info, &ip);
1987                         UNW_DPRINT(0, "unwind.%s: failed to read "
1988                                    "predicate register (ip=0x%lx)\n",
1989                                 __func__, ip);
1990                         return -1;
1991                 }
1992         } while (unw_unwind(info) >= 0);
1993         unw_get_ip(info, &ip);
1994         UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n",
1995                    __func__, ip);
1996         return -1;
1997 }
1998 EXPORT_SYMBOL(unw_unwind_to_user);
1999
2000 static void
2001 init_frame_info (struct unw_frame_info *info, struct task_struct *t,
2002                  struct switch_stack *sw, unsigned long stktop)
2003 {
2004         unsigned long rbslimit, rbstop, stklimit;
2005         STAT(unsigned long start, flags;)
2006
2007         STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
2008
2009         /*
2010          * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
2011          * don't want to do that because it would be slow as each preserved register would
2012          * have to be processed.  Instead, what we do here is zero out the frame info and
2013          * start the unwind process at the function that created the switch_stack frame.
2014          * When a preserved value in switch_stack needs to be accessed, run_script() will
2015          * initialize the appropriate pointer on demand.
2016          */
2017         memset(info, 0, sizeof(*info));
2018
2019         rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
2020         stklimit = (unsigned long) t + IA64_STK_OFFSET;
2021
2022         rbstop   = sw->ar_bspstore;
2023         if (rbstop > stklimit || rbstop < rbslimit)
2024                 rbstop = rbslimit;
2025
2026         if (stktop <= rbstop)
2027                 stktop = rbstop;
2028         if (stktop > stklimit)
2029                 stktop = stklimit;
2030
2031         info->regstk.limit = rbslimit;
2032         info->regstk.top   = rbstop;
2033         info->memstk.limit = stklimit;
2034         info->memstk.top   = stktop;
2035         info->task = t;
2036         info->sw  = sw;
2037         info->sp = info->psp = stktop;
2038         info->pr = sw->pr;
2039         UNW_DPRINT(3, "unwind.%s:\n"
2040                    "  task   0x%lx\n"
2041                    "  rbs = [0x%lx-0x%lx)\n"
2042                    "  stk = [0x%lx-0x%lx)\n"
2043                    "  pr     0x%lx\n"
2044                    "  sw     0x%lx\n"
2045                    "  sp     0x%lx\n",
2046                    __func__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
2047                    info->pr, (unsigned long) info->sw, info->sp);
2048         STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
2049 }
2050
2051 void
2052 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
2053 {
2054         unsigned long sol;
2055
2056         init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
2057         info->cfm_loc = &sw->ar_pfs;
2058         sol = (*info->cfm_loc >> 7) & 0x7f;
2059         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
2060         info->ip = sw->b0;
2061         UNW_DPRINT(3, "unwind.%s:\n"
2062                    "  bsp    0x%lx\n"
2063                    "  sol    0x%lx\n"
2064                    "  ip     0x%lx\n",
2065                    __func__, info->bsp, sol, info->ip);
2066         find_save_locs(info);
2067 }
2068
2069 EXPORT_SYMBOL(unw_init_frame_info);
2070
2071 void
2072 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
2073 {
2074         struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
2075
2076         UNW_DPRINT(1, "unwind.%s\n", __func__);
2077         unw_init_frame_info(info, t, sw);
2078 }
2079 EXPORT_SYMBOL(unw_init_from_blocked_task);
2080
2081 static void
2082 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
2083                    unsigned long gp, const void *table_start, const void *table_end)
2084 {
2085         const struct unw_table_entry *start = table_start, *end = table_end;
2086
2087         table->name = name;
2088         table->segment_base = segment_base;
2089         table->gp = gp;
2090         table->start = segment_base + start[0].start_offset;
2091         table->end = segment_base + end[-1].end_offset;
2092         table->array = start;
2093         table->length = end - start;
2094 }
2095
2096 void *
2097 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2098                       const void *table_start, const void *table_end)
2099 {
2100         const struct unw_table_entry *start = table_start, *end = table_end;
2101         struct unw_table *table;
2102         unsigned long flags;
2103
2104         if (end - start <= 0) {
2105                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2106                            __func__);
2107                 return NULL;
2108         }
2109
2110         table = kmalloc(sizeof(*table), GFP_USER);
2111         if (!table)
2112                 return NULL;
2113
2114         init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2115
2116         spin_lock_irqsave(&unw.lock, flags);
2117         {
2118                 /* keep kernel unwind table at the front (it's searched most commonly): */
2119                 table->next = unw.tables->next;
2120                 unw.tables->next = table;
2121         }
2122         spin_unlock_irqrestore(&unw.lock, flags);
2123
2124         return table;
2125 }
2126
2127 void
2128 unw_remove_unwind_table (void *handle)
2129 {
2130         struct unw_table *table, *prev;
2131         struct unw_script *tmp;
2132         unsigned long flags;
2133         long index;
2134
2135         if (!handle) {
2136                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2137                            __func__);
2138                 return;
2139         }
2140
2141         table = handle;
2142         if (table == &unw.kernel_table) {
2143                 UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2144                            "no-can-do!\n", __func__);
2145                 return;
2146         }
2147
2148         spin_lock_irqsave(&unw.lock, flags);
2149         {
2150                 /* first, delete the table: */
2151
2152                 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2153                         if (prev->next == table)
2154                                 break;
2155                 if (!prev) {
2156                         UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2157                                    __func__, (void *) table);
2158                         spin_unlock_irqrestore(&unw.lock, flags);
2159                         return;
2160                 }
2161                 prev->next = table->next;
2162         }
2163         spin_unlock_irqrestore(&unw.lock, flags);
2164
2165         /* next, remove hash table entries for this table */
2166
2167         for (index = 0; index < UNW_HASH_SIZE; ++index) {
2168                 tmp = unw.cache + unw.hash[index];
2169                 if (unw.hash[index] >= UNW_CACHE_SIZE
2170                     || tmp->ip < table->start || tmp->ip >= table->end)
2171                         continue;
2172
2173                 write_lock(&tmp->lock);
2174                 {
2175                         if (tmp->ip >= table->start && tmp->ip < table->end) {
2176                                 unw.hash[index] = tmp->coll_chain;
2177                                 tmp->ip = 0;
2178                         }
2179                 }
2180                 write_unlock(&tmp->lock);
2181         }
2182
2183         kfree(table);
2184 }
2185
2186 static int __init
2187 create_gate_table (void)
2188 {
2189         const struct unw_table_entry *entry, *start, *end;
2190         unsigned long *lp, segbase = GATE_ADDR;
2191         size_t info_size, size;
2192         char *info;
2193         Elf64_Phdr *punw = NULL, *phdr = (Elf64_Phdr *) (GATE_ADDR + GATE_EHDR->e_phoff);
2194         int i;
2195
2196         for (i = 0; i < GATE_EHDR->e_phnum; ++i, ++phdr)
2197                 if (phdr->p_type == PT_IA_64_UNWIND) {
2198                         punw = phdr;
2199                         break;
2200                 }
2201
2202         if (!punw) {
2203                 printk("%s: failed to find gate DSO's unwind table!\n", __func__);
2204                 return 0;
2205         }
2206
2207         start = (const struct unw_table_entry *) punw->p_vaddr;
2208         end = (struct unw_table_entry *) ((char *) start + punw->p_memsz);
2209         size  = 0;
2210
2211         unw_add_unwind_table("linux-gate.so", segbase, 0, start, end);
2212
2213         for (entry = start; entry < end; ++entry)
2214                 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2215         size += 8;      /* reserve space for "end of table" marker */
2216
2217         unw.gate_table = kmalloc(size, GFP_KERNEL);
2218         if (!unw.gate_table) {
2219                 unw.gate_table_size = 0;
2220                 printk(KERN_ERR "%s: unable to create unwind data for gate page!\n", __func__);
2221                 return 0;
2222         }
2223         unw.gate_table_size = size;
2224
2225         lp = unw.gate_table;
2226         info = (char *) unw.gate_table + size;
2227
2228         for (entry = start; entry < end; ++entry, lp += 3) {
2229                 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2230                 info -= info_size;
2231                 memcpy(info, (char *) segbase + entry->info_offset, info_size);
2232
2233                 lp[0] = segbase + entry->start_offset;          /* start */
2234                 lp[1] = segbase + entry->end_offset;            /* end */
2235                 lp[2] = info - (char *) unw.gate_table;         /* info */
2236         }
2237         *lp = 0;        /* end-of-table marker */
2238         return 0;
2239 }
2240
2241 __initcall(create_gate_table);
2242
2243 void __init
2244 unw_init (void)
2245 {
2246         extern char __gp[];
2247         extern void unw_hash_index_t_is_too_narrow (void);
2248         long i, off;
2249
2250         if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2251                 unw_hash_index_t_is_too_narrow();
2252
2253         unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(CALLER_UNAT);
2254         unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2255         unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_PFS);
2256         unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2257         unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(CALLER_UNAT);
2258         unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2259         unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2260         unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2261         for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2262                 unw.sw_off[unw.preg_index[i]] = off;
2263         for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2264                 unw.sw_off[unw.preg_index[i]] = off;
2265         for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2266                 unw.sw_off[unw.preg_index[i]] = off;
2267         for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2268                 unw.sw_off[unw.preg_index[i]] = off;
2269
2270         for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2271                 if (i > 0)
2272                         unw.cache[i].lru_chain = (i - 1);
2273                 unw.cache[i].coll_chain = -1;
2274                 rwlock_init(&unw.cache[i].lock);
2275         }
2276         unw.lru_head = UNW_CACHE_SIZE - 1;
2277         unw.lru_tail = 0;
2278
2279         init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) __gp,
2280                           __start_unwind, __end_unwind);
2281 }
2282
2283 /*
2284  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2285  *
2286  *      This system call has been deprecated.  The new and improved way to get
2287  *      at the kernel's unwind info is via the gate DSO.  The address of the
2288  *      ELF header for this DSO is passed to user-level via AT_SYSINFO_EHDR.
2289  *
2290  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2291  *
2292  * This system call copies the unwind data into the buffer pointed to by BUF and returns
2293  * the size of the unwind data.  If BUF_SIZE is smaller than the size of the unwind data
2294  * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2295  * unwind data.
2296  *
2297  * The first portion of the unwind data contains an unwind table and rest contains the
2298  * associated unwind info (in no particular order).  The unwind table consists of a table
2299  * of entries of the form:
2300  *
2301  *      u64 start;      (64-bit address of start of function)
2302  *      u64 end;        (64-bit address of start of function)
2303  *      u64 info;       (BUF-relative offset to unwind info)
2304  *
2305  * The end of the unwind table is indicated by an entry with a START address of zero.
2306  *
2307  * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2308  * on the format of the unwind info.
2309  *
2310  * ERRORS
2311  *      EFAULT  BUF points outside your accessible address space.
2312  */
2313 asmlinkage long
2314 sys_getunwind (void __user *buf, size_t buf_size)
2315 {
2316         if (buf && buf_size >= unw.gate_table_size)
2317                 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2318                         return -EFAULT;
2319         return unw.gate_table_size;
2320 }