arch/ia64/kernel/unwind.c

   1 /*
   2  * Copyright (C) 1999-2004 Hewlett-Packard Co
   3  *      David Mosberger-Tang <davidm@hpl.hp.com>
   4  * Copyright (C) 2003 Fenghua Yu <fenghua.yu@intel.com>
   5  *      - Change pt_regs_off() to make it less dependant on pt_regs structure.
   6  */
   7 /*
   8  * This file implements call frame unwind support for the Linux
   9  * kernel.  Parsing and processing the unwind information is
  10  * time-consuming, so this implementation translates the unwind
  11  * descriptors into unwind scripts.  These scripts are very simple
  12  * (basically a sequence of assignments) and efficient to execute.
  13  * They are cached for later re-use.  Each script is specific for a
  14  * given instruction pointer address and the set of predicate values
  15  * that the script depends on (most unwind descriptors are
  16  * unconditional and scripts often do not depend on predicates at
  17  * all).  This code is based on the unwind conventions described in
  18  * the "IA-64 Software Conventions and Runtime Architecture" manual.
  19  *
  20  * SMP conventions:
  21  *      o updates to the global unwind data (in structure "unw") are serialized
  22  *        by the unw.lock spinlock
  23  *      o each unwind script has its own read-write lock; a thread must acquire
  24  *        a read lock before executing a script and must acquire a write lock
  25  *        before modifying a script
  26  *      o if both the unw.lock spinlock and a script's read-write lock must be
  27  *        acquired, then the read-write lock must be acquired first.
  28  */
  29 #include <linux/module.h>
  30 #include <linux/bootmem.h>
  31 #include <linux/elf.h>
  32 #include <linux/kernel.h>
  33 #include <linux/sched.h>
  34 #include <linux/slab.h>
  35
  36 #include <asm/unwind.h>
  37
  38 #include <asm/delay.h>
  39 #include <asm/page.h>
  40 #include <asm/ptrace.h>
  41 #include <asm/ptrace_offsets.h>
  42 #include <asm/rse.h>
  43 #include <asm/sections.h>
  44 #include <asm/system.h>
  45 #include <asm/uaccess.h>
  46
  47 #include "entry.h"
  48 #include "unwind_i.h"
  49
  50 #define UNW_LOG_CACHE_SIZE      7       /* each unw_script is ~256 bytes in size */
  51 #define UNW_CACHE_SIZE          (1 << UNW_LOG_CACHE_SIZE)
  52
  53 #define UNW_LOG_HASH_SIZE       (UNW_LOG_CACHE_SIZE + 1)
  54 #define UNW_HASH_SIZE           (1 << UNW_LOG_HASH_SIZE)
  55
  56 #define UNW_STATS       0       /* WARNING: this disabled interrupts for long time-spans!! */
  57
  58 #ifdef UNW_DEBUG
  59   static unsigned int unw_debug_level = UNW_DEBUG;
  60 #  define UNW_DEBUG_ON(n)       unw_debug_level >= n
  61    /* Do not code a printk level, not all debug lines end in newline */
  62 #  define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
  63 #  define inline
  64 #else /* !UNW_DEBUG */
  65 #  define UNW_DEBUG_ON(n)  0
  66 #  define UNW_DPRINT(n, ...)
  67 #endif /* UNW_DEBUG */
  68
  69 #if UNW_STATS
  70 # define STAT(x...)     x
  71 #else
  72 # define STAT(x...)
  73 #endif
  74
  75 #define alloc_reg_state()       kmalloc(sizeof(struct unw_reg_state), GFP_ATOMIC)
  76 #define free_reg_state(usr)     kfree(usr)
  77 #define alloc_labeled_state()   kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
  78 #define free_labeled_state(usr) kfree(usr)
  79
  80 typedef unsigned long unw_word;
  81 typedef unsigned char unw_hash_index_t;
  82
  83 static struct {
  84         spinlock_t lock;                        /* spinlock for unwind data */
  85
  86         /* list of unwind tables (one per load-module) */
  87         struct unw_table *tables;
  88
  89         unsigned long r0;                       /* constant 0 for r0 */
  90
  91         /* table of registers that prologues can save (and order in which they're saved): */
  92         const unsigned char save_order[8];
  93
  94         /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
  95         unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
  96
  97         unsigned short lru_head;                /* index of lead-recently used script */
  98         unsigned short lru_tail;                /* index of most-recently used script */
  99
 100         /* index into unw_frame_info for preserved register i */
 101         unsigned short preg_index[UNW_NUM_REGS];
 102
 103         short pt_regs_offsets[32];
 104
 105         /* unwind table for the kernel: */
 106         struct unw_table kernel_table;
 107
 108         /* unwind table describing the gate page (kernel code that is mapped into user space): */
 109         size_t gate_table_size;
 110         unsigned long *gate_table;
 111
 112         /* hash table that maps instruction pointer to script index: */
 113         unsigned short hash[UNW_HASH_SIZE];
 114
 115         /* script cache: */
 116         struct unw_script cache[UNW_CACHE_SIZE];
 117
 118 # ifdef UNW_DEBUG
 119         const char *preg_name[UNW_NUM_REGS];
 120 # endif
 121 # if UNW_STATS
 122         struct {
 123                 struct {
 124                         int lookups;
 125                         int hinted_hits;
 126                         int normal_hits;
 127                         int collision_chain_traversals;
 128                 } cache;
 129                 struct {
 130                         unsigned long build_time;
 131                         unsigned long run_time;
 132                         unsigned long parse_time;
 133                         int builds;
 134                         int news;
 135                         int collisions;
 136                         int runs;
 137                 } script;
 138                 struct {
 139                         unsigned long init_time;
 140                         unsigned long unwind_time;
 141                         int inits;
 142                         int unwinds;
 143                 } api;
 144         } stat;
 145 # endif
 146 } unw = {
 147         .tables = &unw.kernel_table,
 148         .lock = SPIN_LOCK_UNLOCKED,
 149         .save_order = {
 150                 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
 151                 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
 152         },
 153         .preg_index = {
 154                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_GR */
 155                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_MEM */
 156                 offsetof(struct unw_frame_info, bsp_loc)/8,
 157                 offsetof(struct unw_frame_info, bspstore_loc)/8,
 158                 offsetof(struct unw_frame_info, pfs_loc)/8,
 159                 offsetof(struct unw_frame_info, rnat_loc)/8,
 160                 offsetof(struct unw_frame_info, psp)/8,
 161                 offsetof(struct unw_frame_info, rp_loc)/8,
 162                 offsetof(struct unw_frame_info, r4)/8,
 163                 offsetof(struct unw_frame_info, r5)/8,
 164                 offsetof(struct unw_frame_info, r6)/8,
 165                 offsetof(struct unw_frame_info, r7)/8,
 166                 offsetof(struct unw_frame_info, unat_loc)/8,
 167                 offsetof(struct unw_frame_info, pr_loc)/8,
 168                 offsetof(struct unw_frame_info, lc_loc)/8,
 169                 offsetof(struct unw_frame_info, fpsr_loc)/8,
 170                 offsetof(struct unw_frame_info, b1_loc)/8,
 171                 offsetof(struct unw_frame_info, b2_loc)/8,
 172                 offsetof(struct unw_frame_info, b3_loc)/8,
 173                 offsetof(struct unw_frame_info, b4_loc)/8,
 174                 offsetof(struct unw_frame_info, b5_loc)/8,
 175                 offsetof(struct unw_frame_info, f2_loc)/8,
 176                 offsetof(struct unw_frame_info, f3_loc)/8,
 177                 offsetof(struct unw_frame_info, f4_loc)/8,
 178                 offsetof(struct unw_frame_info, f5_loc)/8,
 179                 offsetof(struct unw_frame_info, fr_loc[16 - 16])/8,
 180                 offsetof(struct unw_frame_info, fr_loc[17 - 16])/8,
 181                 offsetof(struct unw_frame_info, fr_loc[18 - 16])/8,
 182                 offsetof(struct unw_frame_info, fr_loc[19 - 16])/8,
 183                 offsetof(struct unw_frame_info, fr_loc[20 - 16])/8,
 184                 offsetof(struct unw_frame_info, fr_loc[21 - 16])/8,
 185                 offsetof(struct unw_frame_info, fr_loc[22 - 16])/8,
 186                 offsetof(struct unw_frame_info, fr_loc[23 - 16])/8,
 187                 offsetof(struct unw_frame_info, fr_loc[24 - 16])/8,
 188                 offsetof(struct unw_frame_info, fr_loc[25 - 16])/8,
 189                 offsetof(struct unw_frame_info, fr_loc[26 - 16])/8,
 190                 offsetof(struct unw_frame_info, fr_loc[27 - 16])/8,
 191                 offsetof(struct unw_frame_info, fr_loc[28 - 16])/8,
 192                 offsetof(struct unw_frame_info, fr_loc[29 - 16])/8,
 193                 offsetof(struct unw_frame_info, fr_loc[30 - 16])/8,
 194                 offsetof(struct unw_frame_info, fr_loc[31 - 16])/8,
 195         },
 196         .pt_regs_offsets = {
 197                 [0] = -1,
 198                 offsetof(struct pt_regs,  r1),
 199                 offsetof(struct pt_regs,  r2),
 200                 offsetof(struct pt_regs,  r3),
 201                 [4] = -1, [5] = -1, [6] = -1, [7] = -1,
 202                 offsetof(struct pt_regs,  r8),
 203                 offsetof(struct pt_regs,  r9),
 204                 offsetof(struct pt_regs, r10),
 205                 offsetof(struct pt_regs, r11),
 206                 offsetof(struct pt_regs, r12),
 207                 offsetof(struct pt_regs, r13),
 208                 offsetof(struct pt_regs, r14),
 209                 offsetof(struct pt_regs, r15),
 210                 offsetof(struct pt_regs, r16),
 211                 offsetof(struct pt_regs, r17),
 212                 offsetof(struct pt_regs, r18),
 213                 offsetof(struct pt_regs, r19),
 214                 offsetof(struct pt_regs, r20),
 215                 offsetof(struct pt_regs, r21),
 216                 offsetof(struct pt_regs, r22),
 217                 offsetof(struct pt_regs, r23),
 218                 offsetof(struct pt_regs, r24),
 219                 offsetof(struct pt_regs, r25),
 220                 offsetof(struct pt_regs, r26),
 221                 offsetof(struct pt_regs, r27),
 222                 offsetof(struct pt_regs, r28),
 223                 offsetof(struct pt_regs, r29),
 224                 offsetof(struct pt_regs, r30),
 225                 offsetof(struct pt_regs, r31),
 226         },
 227         .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
 228 #ifdef UNW_DEBUG
 229         .preg_name = {
 230                 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
 231                 "r4", "r5", "r6", "r7",
 232                 "ar.unat", "pr", "ar.lc", "ar.fpsr",
 233                 "b1", "b2", "b3", "b4", "b5",
 234                 "f2", "f3", "f4", "f5",
 235                 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
 236                 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
 237         }
 238 #endif
 239 };
 240
 241 static inline int
 242 read_only (void *addr)
 243 {
 244         return (unsigned long) ((char *) addr - (char *) &unw.r0) < sizeof(unw.r0);
 245 }
 246
 247 /*
 248  * Returns offset of rREG in struct pt_regs.
 249  */
 250 static inline unsigned long
 251 pt_regs_off (unsigned long reg)
 252 {
 253         short off = -1;
 254
 255         if (reg < ARRAY_SIZE(unw.pt_regs_offsets))
 256                 off = unw.pt_regs_offsets[reg];
 257
 258         if (off < 0) {
 259                 UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __FUNCTION__, reg);
 260                 off = 0;
 261         }
 262         return (unsigned long) off;
 263 }
 264
 265 static inline struct pt_regs *
 266 get_scratch_regs (struct unw_frame_info *info)
 267 {
 268         if (!info->pt) {
 269                 /* This should not happen with valid unwind info.  */
 270                 UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __FUNCTION__);
 271                 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
 272                         info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
 273                 else
 274                         info->pt = info->sp - 16;
 275         }
 276         UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __FUNCTION__, info->sp, info->pt);
 277         return (struct pt_regs *) info->pt;
 278 }
 279
 280 /* Unwind accessors.  */
 281
 282 int
 283 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
 284 {
 285         unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
 286         struct unw_ireg *ireg;
 287         struct pt_regs *pt;
 288
 289         if ((unsigned) regnum - 1 >= 127) {
 290                 if (regnum == 0 && !write) {
 291                         *val = 0;       /* read r0 always returns 0 */
 292                         *nat = 0;
 293                         return 0;
 294                 }
 295                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
 296                            __FUNCTION__, regnum);
 297                 return -1;
 298         }
 299
 300         if (regnum < 32) {
 301                 if (regnum >= 4 && regnum <= 7) {
 302                         /* access a preserved register */
 303                         ireg = &info->r4 + (regnum - 4);
 304                         addr = ireg->loc;
 305                         if (addr) {
 306                                 nat_addr = addr + ireg->nat.off;
 307                                 switch (ireg->nat.type) {
 308                                       case UNW_NAT_VAL:
 309                                         /* simulate getf.sig/setf.sig */
 310                                         if (write) {
 311                                                 if (*nat) {
 312                                                         /* write NaTVal and be done with it */
 313                                                         addr[0] = 0;
 314                                                         addr[1] = 0x1fffe;
 315                                                         return 0;
 316                                                 }
 317                                                 addr[1] = 0x1003e;
 318                                         } else {
 319                                                 if (addr[0] == 0 && addr[1] == 0x1ffe) {
 320                                                         /* return NaT and be done with it */
 321                                                         *val = 0;
 322                                                         *nat = 1;
 323                                                         return 0;
 324                                                 }
 325                                         }
 326                                         /* fall through */
 327                                       case UNW_NAT_NONE:
 328                                         dummy_nat = 0;
 329                                         nat_addr = &dummy_nat;
 330                                         break;
 331
 332                                       case UNW_NAT_MEMSTK:
 333                                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 334                                         break;
 335
 336                                       case UNW_NAT_REGSTK:
 337                                         nat_addr = ia64_rse_rnat_addr(addr);
 338                                         if ((unsigned long) addr < info->regstk.limit
 339                                             || (unsigned long) addr >= info->regstk.top)
 340                                         {
 341                                                 UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
 342                                                         "[0x%lx-0x%lx)\n",
 343                                                         __FUNCTION__, (void *) addr,
 344                                                         info->regstk.limit,
 345                                                         info->regstk.top);
 346                                                 return -1;
 347                                         }
 348                                         if ((unsigned long) nat_addr >= info->regstk.top)
 349                                                 nat_addr = &info->sw->ar_rnat;
 350                                         nat_mask = (1UL << ia64_rse_slot_num(addr));
 351                                         break;
 352                                 }
 353                         } else {
 354                                 addr = &info->sw->r4 + (regnum - 4);
 355                                 nat_addr = &info->sw->ar_unat;
 356                                 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 357                         }
 358                 } else {
 359                         /* access a scratch register */
 360                         pt = get_scratch_regs(info);
 361                         addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
 362                         if (info->pri_unat_loc)
 363                                 nat_addr = info->pri_unat_loc;
 364                         else
 365                                 nat_addr = &info->sw->caller_unat;
 366                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 367                 }
 368         } else {
 369                 /* access a stacked register */
 370                 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
 371                 nat_addr = ia64_rse_rnat_addr(addr);
 372                 if ((unsigned long) addr < info->regstk.limit
 373                     || (unsigned long) addr >= info->regstk.top)
 374                 {
 375                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
 376                                    "of rbs\n",  __FUNCTION__);
 377                         return -1;
 378                 }
 379                 if ((unsigned long) nat_addr >= info->regstk.top)
 380                         nat_addr = &info->sw->ar_rnat;
 381                 nat_mask = (1UL << ia64_rse_slot_num(addr));
 382         }
 383
 384         if (write) {
 385                 if (read_only(addr)) {
 386                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 387                                 __FUNCTION__);
 388                 } else {
 389                         *addr = *val;
 390                         if (*nat)
 391                                 *nat_addr |= nat_mask;
 392                         else
 393                                 *nat_addr &= ~nat_mask;
 394                 }
 395         } else {
 396                 if ((*nat_addr & nat_mask) == 0) {
 397                         *val = *addr;
 398                         *nat = 0;
 399                 } else {
 400                         *val = 0;       /* if register is a NaT, *addr may contain kernel data! */
 401                         *nat = 1;
 402                 }
 403         }
 404         return 0;
 405 }
 406 EXPORT_SYMBOL(unw_access_gr);
 407
 408 int
 409 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 410 {
 411         unsigned long *addr;
 412         struct pt_regs *pt;
 413
 414         switch (regnum) {
 415                 /* scratch: */
 416               case 0: pt = get_scratch_regs(info); addr = &pt->b0; break;
 417               case 6: pt = get_scratch_regs(info); addr = &pt->b6; break;
 418               case 7: pt = get_scratch_regs(info); addr = &pt->b7; break;
 419
 420                 /* preserved: */
 421               case 1: case 2: case 3: case 4: case 5:
 422                 addr = *(&info->b1_loc + (regnum - 1));
 423                 if (!addr)
 424                         addr = &info->sw->b1 + (regnum - 1);
 425                 break;
 426
 427               default:
 428                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
 429                            __FUNCTION__, regnum);
 430                 return -1;
 431         }
 432         if (write)
 433                 if (read_only(addr)) {
 434                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 435                                 __FUNCTION__);
 436                 } else
 437                         *addr = *val;
 438         else
 439                 *val = *addr;
 440         return 0;
 441 }
 442 EXPORT_SYMBOL(unw_access_br);
 443
 444 int
 445 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
 446 {
 447         struct ia64_fpreg *addr = NULL;
 448         struct pt_regs *pt;
 449
 450         if ((unsigned) (regnum - 2) >= 126) {
 451                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
 452                            __FUNCTION__, regnum);
 453                 return -1;
 454         }
 455
 456         if (regnum <= 5) {
 457                 addr = *(&info->f2_loc + (regnum - 2));
 458                 if (!addr)
 459                         addr = &info->sw->f2 + (regnum - 2);
 460         } else if (regnum <= 15) {
 461                 if (regnum <= 11) {
 462                         pt = get_scratch_regs(info);
 463                         addr = &pt->f6  + (regnum - 6);
 464                 }
 465                 else
 466                         addr = &info->sw->f12 + (regnum - 12);
 467         } else if (regnum <= 31) {
 468                 addr = info->fr_loc[regnum - 16];
 469                 if (!addr)
 470                         addr = &info->sw->f16 + (regnum - 16);
 471         } else {
 472                 struct task_struct *t = info->task;
 473
 474                 if (write)
 475                         ia64_sync_fph(t);
 476                 else
 477                         ia64_flush_fph(t);
 478                 addr = t->thread.fph + (regnum - 32);
 479         }
 480
 481         if (write)
 482                 if (read_only(addr)) {
 483                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 484                                 __FUNCTION__);
 485                 } else
 486                         *addr = *val;
 487         else
 488                 *val = *addr;
 489         return 0;
 490 }
 491 EXPORT_SYMBOL(unw_access_fr);
 492
 493 int
 494 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 495 {
 496         unsigned long *addr;
 497         struct pt_regs *pt;
 498
 499         switch (regnum) {
 500               case UNW_AR_BSP:
 501                 addr = info->bsp_loc;
 502                 if (!addr)
 503                         addr = &info->sw->ar_bspstore;
 504                 break;
 505
 506               case UNW_AR_BSPSTORE:
 507                 addr = info->bspstore_loc;
 508                 if (!addr)
 509                         addr = &info->sw->ar_bspstore;
 510                 break;
 511
 512               case UNW_AR_PFS:
 513                 addr = info->pfs_loc;
 514                 if (!addr)
 515                         addr = &info->sw->ar_pfs;
 516                 break;
 517
 518               case UNW_AR_RNAT:
 519                 addr = info->rnat_loc;
 520                 if (!addr)
 521                         addr = &info->sw->ar_rnat;
 522                 break;
 523
 524               case UNW_AR_UNAT:
 525                 addr = info->unat_loc;
 526                 if (!addr)
 527                         addr = &info->sw->caller_unat;
 528                 break;
 529
 530               case UNW_AR_LC:
 531                 addr = info->lc_loc;
 532                 if (!addr)
 533                         addr = &info->sw->ar_lc;
 534                 break;
 535
 536               case UNW_AR_EC:
 537                 if (!info->cfm_loc)
 538                         return -1;
 539                 if (write)
 540                         *info->cfm_loc =
 541                                 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
 542                 else
 543                         *val = (*info->cfm_loc >> 52) & 0x3f;
 544                 return 0;
 545
 546               case UNW_AR_FPSR:
 547                 addr = info->fpsr_loc;
 548                 if (!addr)
 549                         addr = &info->sw->ar_fpsr;
 550                 break;
 551
 552               case UNW_AR_RSC:
 553                 pt = get_scratch_regs(info);
 554                 addr = &pt->ar_rsc;
 555                 break;
 556
 557               case UNW_AR_CCV:
 558                 pt = get_scratch_regs(info);
 559                 addr = &pt->ar_ccv;
 560                 break;
 561
 562               case UNW_AR_CSD:
 563                 pt = get_scratch_regs(info);
 564                 addr = &pt->ar_csd;
 565                 break;
 566
 567               case UNW_AR_SSD:
 568                 pt = get_scratch_regs(info);
 569                 addr = &pt->ar_ssd;
 570                 break;
 571
 572               default:
 573                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
 574                            __FUNCTION__, regnum);
 575                 return -1;
 576         }
 577
 578         if (write) {
 579                 if (read_only(addr)) {
 580                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 581                                 __FUNCTION__);
 582                 } else
 583                         *addr = *val;
 584         } else
 585                 *val = *addr;
 586         return 0;
 587 }
 588 EXPORT_SYMBOL(unw_access_ar);
 589
 590 int
 591 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
 592 {
 593         unsigned long *addr;
 594
 595         addr = info->pr_loc;
 596         if (!addr)
 597                 addr = &info->sw->pr;
 598
 599         if (write) {
 600                 if (read_only(addr)) {
 601                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 602                                 __FUNCTION__);
 603                 } else
 604                         *addr = *val;
 605         } else
 606                 *val = *addr;
 607         return 0;
 608 }
 609 EXPORT_SYMBOL(unw_access_pr);
 610
 611 \f
 612 /* Routines to manipulate the state stack.  */
 613
 614 static inline void
 615 push (struct unw_state_record *sr)
 616 {
 617         struct unw_reg_state *rs;
 618
 619         rs = alloc_reg_state();
 620         if (!rs) {
 621                 printk(KERN_ERR "unwind: cannot stack reg state!\n");
 622                 return;
 623         }
 624         memcpy(rs, &sr->curr, sizeof(*rs));
 625         sr->curr.next = rs;
 626 }
 627
 628 static void
 629 pop (struct unw_state_record *sr)
 630 {
 631         struct unw_reg_state *rs = sr->curr.next;
 632
 633         if (!rs) {
 634                 printk(KERN_ERR "unwind: stack underflow!\n");
 635                 return;
 636         }
 637         memcpy(&sr->curr, rs, sizeof(*rs));
 638         free_reg_state(rs);
 639 }
 640
 641 /* Make a copy of the state stack.  Non-recursive to avoid stack overflows.  */
 642 static struct unw_reg_state *
 643 dup_state_stack (struct unw_reg_state *rs)
 644 {
 645         struct unw_reg_state *copy, *prev = NULL, *first = NULL;
 646
 647         while (rs) {
 648                 copy = alloc_reg_state();
 649                 if (!copy) {
 650                         printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
 651                         return NULL;
 652                 }
 653                 memcpy(copy, rs, sizeof(*copy));
 654                 if (first)
 655                         prev->next = copy;
 656                 else
 657                         first = copy;
 658                 rs = rs->next;
 659                 prev = copy;
 660         }
 661         return first;
 662 }
 663
 664 /* Free all stacked register states (but not RS itself).  */
 665 static void
 666 free_state_stack (struct unw_reg_state *rs)
 667 {
 668         struct unw_reg_state *p, *next;
 669
 670         for (p = rs->next; p != NULL; p = next) {
 671                 next = p->next;
 672                 free_reg_state(p);
 673         }
 674         rs->next = NULL;
 675 }
 676 \f
 677 /* Unwind decoder routines */
 678
 679 static enum unw_register_index __attribute_const__
 680 decode_abreg (unsigned char abreg, int memory)
 681 {
 682         switch (abreg) {
 683               case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
 684               case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
 685               case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
 686               case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
 687               case 0x60: return UNW_REG_PR;
 688               case 0x61: return UNW_REG_PSP;
 689               case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
 690               case 0x63: return UNW_REG_RP;
 691               case 0x64: return UNW_REG_BSP;
 692               case 0x65: return UNW_REG_BSPSTORE;
 693               case 0x66: return UNW_REG_RNAT;
 694               case 0x67: return UNW_REG_UNAT;
 695               case 0x68: return UNW_REG_FPSR;
 696               case 0x69: return UNW_REG_PFS;
 697               case 0x6a: return UNW_REG_LC;
 698               default:
 699                 break;
 700         }
 701         UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __FUNCTION__, abreg);
 702         return UNW_REG_LC;
 703 }
 704
 705 static void
 706 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
 707 {
 708         reg->val = val;
 709         reg->where = where;
 710         if (reg->when == UNW_WHEN_NEVER)
 711                 reg->when = when;
 712 }
 713
 714 static void
 715 alloc_spill_area (unsigned long *offp, unsigned long regsize,
 716                   struct unw_reg_info *lo, struct unw_reg_info *hi)
 717 {
 718         struct unw_reg_info *reg;
 719
 720         for (reg = hi; reg >= lo; --reg) {
 721                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 722                         reg->where = UNW_WHERE_PSPREL;
 723                         *offp -= regsize;
 724                         reg->val = *offp;
 725                 }
 726         }
 727 }
 728
 729 static inline void
 730 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
 731 {
 732         struct unw_reg_info *reg;
 733
 734         for (reg = *regp; reg <= lim; ++reg) {
 735                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 736                         reg->when = t;
 737                         *regp = reg + 1;
 738                         return;
 739                 }
 740         }
 741         UNW_DPRINT(0, "unwind.%s: excess spill!\n",  __FUNCTION__);
 742 }
 743
 744 static inline void
 745 finish_prologue (struct unw_state_record *sr)
 746 {
 747         struct unw_reg_info *reg;
 748         unsigned long off;
 749         int i;
 750
 751         /*
 752          * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
 753          * for Using Unwind Descriptors", rule 3):
 754          */
 755         for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
 756                 reg = sr->curr.reg + unw.save_order[i];
 757                 if (reg->where == UNW_WHERE_GR_SAVE) {
 758                         reg->where = UNW_WHERE_GR;
 759                         reg->val = sr->gr_save_loc++;
 760                 }
 761         }
 762
 763         /*
 764          * Next, compute when the fp, general, and branch registers get
 765          * saved.  This must come before alloc_spill_area() because
 766          * we need to know which registers are spilled to their home
 767          * locations.
 768          */
 769         if (sr->imask) {
 770                 unsigned char kind, mask = 0, *cp = sr->imask;
 771                 int t;
 772                 static const unsigned char limit[3] = {
 773                         UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
 774                 };
 775                 struct unw_reg_info *(regs[3]);
 776
 777                 regs[0] = sr->curr.reg + UNW_REG_F2;
 778                 regs[1] = sr->curr.reg + UNW_REG_R4;
 779                 regs[2] = sr->curr.reg + UNW_REG_B1;
 780
 781                 for (t = 0; t < sr->region_len; ++t) {
 782                         if ((t & 3) == 0)
 783                                 mask = *cp++;
 784                         kind = (mask >> 2*(3-(t & 3))) & 3;
 785                         if (kind > 0)
 786                                 spill_next_when(&regs[kind - 1], sr->curr.reg + limit[kind - 1],
 787                                                 sr->region_start + t);
 788                 }
 789         }
 790         /*
 791          * Next, lay out the memory stack spill area:
 792          */
 793         if (sr->any_spills) {
 794                 off = sr->spill_offset;
 795                 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
 796                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
 797                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
 798         }
 799 }
 800
 801 /*
 802  * Region header descriptors.
 803  */
 804
 805 static void
 806 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
 807                struct unw_state_record *sr)
 808 {
 809         int i, region_start;
 810
 811         if (!(sr->in_body || sr->first_region))
 812                 finish_prologue(sr);
 813         sr->first_region = 0;
 814
 815         /* check if we're done: */
 816         if (sr->when_target < sr->region_start + sr->region_len) {
 817                 sr->done = 1;
 818                 return;
 819         }
 820
 821         region_start = sr->region_start + sr->region_len;
 822
 823         for (i = 0; i < sr->epilogue_count; ++i)
 824                 pop(sr);
 825         sr->epilogue_count = 0;
 826         sr->epilogue_start = UNW_WHEN_NEVER;
 827
 828         sr->region_start = region_start;
 829         sr->region_len = rlen;
 830         sr->in_body = body;
 831
 832         if (!body) {
 833                 push(sr);
 834
 835                 for (i = 0; i < 4; ++i) {
 836                         if (mask & 0x8)
 837                                 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
 838                                         sr->region_start + sr->region_len - 1, grsave++);
 839                         mask <<= 1;
 840                 }
 841                 sr->gr_save_loc = grsave;
 842                 sr->any_spills = 0;
 843                 sr->imask = NULL;
 844                 sr->spill_offset = 0x10;        /* default to psp+16 */
 845         }
 846 }
 847
 848 /*
 849  * Prologue descriptors.
 850  */
 851
 852 static inline void
 853 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
 854 {
 855         if (abi == 3 && context == 'i') {
 856                 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
 857                 UNW_DPRINT(3, "unwind.%s: interrupt frame\n",  __FUNCTION__);
 858         }
 859         else
 860                 UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
 861                                 __FUNCTION__, abi, context);
 862 }
 863
 864 static inline void
 865 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
 866 {
 867         int i;
 868
 869         for (i = 0; i < 5; ++i) {
 870                 if (brmask & 1)
 871                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
 872                                 sr->region_start + sr->region_len - 1, gr++);
 873                 brmask >>= 1;
 874         }
 875 }
 876
 877 static inline void
 878 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
 879 {
 880         int i;
 881
 882         for (i = 0; i < 5; ++i) {
 883                 if (brmask & 1) {
 884                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
 885                                 sr->region_start + sr->region_len - 1, 0);
 886                         sr->any_spills = 1;
 887                 }
 888                 brmask >>= 1;
 889         }
 890 }
 891
 892 static inline void
 893 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
 894 {
 895         int i;
 896
 897         for (i = 0; i < 4; ++i) {
 898                 if ((grmask & 1) != 0) {
 899                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 900                                 sr->region_start + sr->region_len - 1, 0);
 901                         sr->any_spills = 1;
 902                 }
 903                 grmask >>= 1;
 904         }
 905         for (i = 0; i < 20; ++i) {
 906                 if ((frmask & 1) != 0) {
 907                         int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
 908                         set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
 909                                 sr->region_start + sr->region_len - 1, 0);
 910                         sr->any_spills = 1;
 911                 }
 912                 frmask >>= 1;
 913         }
 914 }
 915
 916 static inline void
 917 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
 918 {
 919         int i;
 920
 921         for (i = 0; i < 4; ++i) {
 922                 if ((frmask & 1) != 0) {
 923                         set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
 924                                 sr->region_start + sr->region_len - 1, 0);
 925                         sr->any_spills = 1;
 926                 }
 927                 frmask >>= 1;
 928         }
 929 }
 930
 931 static inline void
 932 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
 933 {
 934         int i;
 935
 936         for (i = 0; i < 4; ++i) {
 937                 if ((grmask & 1) != 0)
 938                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
 939                                 sr->region_start + sr->region_len - 1, gr++);
 940                 grmask >>= 1;
 941         }
 942 }
 943
 944 static inline void
 945 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
 946 {
 947         int i;
 948
 949         for (i = 0; i < 4; ++i) {
 950                 if ((grmask & 1) != 0) {
 951                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 952                                 sr->region_start + sr->region_len - 1, 0);
 953                         sr->any_spills = 1;
 954                 }
 955                 grmask >>= 1;
 956         }
 957 }
 958
 959 static inline void
 960 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
 961 {
 962         set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
 963                 sr->region_start + min_t(int, t, sr->region_len - 1), 16*size);
 964 }
 965
 966 static inline void
 967 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
 968 {
 969         sr->curr.reg[UNW_REG_PSP].when = sr->region_start + min_t(int, t, sr->region_len - 1);
 970 }
 971
 972 static inline void
 973 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
 974 {
 975         set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
 976 }
 977
 978 static inline void
 979 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
 980 {
 981         set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
 982                 0x10 - 4*pspoff);
 983 }
 984
 985 static inline void
 986 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
 987 {
 988         set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
 989                 4*spoff);
 990 }
 991
 992 static inline void
 993 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
 994 {
 995         sr->return_link_reg = dst;
 996 }
 997
 998 static inline void
 999 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
1000 {
1001         struct unw_reg_info *reg = sr->curr.reg + regnum;
1002
1003         if (reg->where == UNW_WHERE_NONE)
1004                 reg->where = UNW_WHERE_GR_SAVE;
1005         reg->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1006 }
1007
1008 static inline void
1009 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
1010 {
1011         sr->spill_offset = 0x10 - 4*pspoff;
1012 }
1013
1014 static inline unsigned char *
1015 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
1016 {
1017         sr->imask = imaskp;
1018         return imaskp + (2*sr->region_len + 7)/8;
1019 }
1020
1021 /*
1022  * Body descriptors.
1023  */
1024 static inline void
1025 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
1026 {
1027         sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
1028         sr->epilogue_count = ecount + 1;
1029 }
1030
1031 static inline void
1032 desc_copy_state (unw_word label, struct unw_state_record *sr)
1033 {
1034         struct unw_labeled_state *ls;
1035
1036         for (ls = sr->labeled_states; ls; ls = ls->next) {
1037                 if (ls->label == label) {
1038                         free_state_stack(&sr->curr);
1039                         memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
1040                         sr->curr.next = dup_state_stack(ls->saved_state.next);
1041                         return;
1042                 }
1043         }
1044         printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
1045 }
1046
1047 static inline void
1048 desc_label_state (unw_word label, struct unw_state_record *sr)
1049 {
1050         struct unw_labeled_state *ls;
1051
1052         ls = alloc_labeled_state();
1053         if (!ls) {
1054                 printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
1055                 return;
1056         }
1057         ls->label = label;
1058         memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
1059         ls->saved_state.next = dup_state_stack(sr->curr.next);
1060
1061         /* insert into list of labeled states: */
1062         ls->next = sr->labeled_states;
1063         sr->labeled_states = ls;
1064 }
1065
1066 /*
1067  * General descriptors.
1068  */
1069
1070 static inline int
1071 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1072 {
1073         if (sr->when_target <= sr->region_start + min_t(int, t, sr->region_len - 1))
1074                 return 0;
1075         if (qp > 0) {
1076                 if ((sr->pr_val & (1UL << qp)) == 0)
1077                         return 0;
1078                 sr->pr_mask |= (1UL << qp);
1079         }
1080         return 1;
1081 }
1082
1083 static inline void
1084 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1085 {
1086         struct unw_reg_info *r;
1087
1088         if (!desc_is_active(qp, t, sr))
1089                 return;
1090
1091         r = sr->curr.reg + decode_abreg(abreg, 0);
1092         r->where = UNW_WHERE_NONE;
1093         r->when = UNW_WHEN_NEVER;
1094         r->val = 0;
1095 }
1096
1097 static inline void
1098 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1099                      unsigned char ytreg, struct unw_state_record *sr)
1100 {
1101         enum unw_where where = UNW_WHERE_GR;
1102         struct unw_reg_info *r;
1103
1104         if (!desc_is_active(qp, t, sr))
1105                 return;
1106
1107         if (x)
1108                 where = UNW_WHERE_BR;
1109         else if (ytreg & 0x80)
1110                 where = UNW_WHERE_FR;
1111
1112         r = sr->curr.reg + decode_abreg(abreg, 0);
1113         r->where = where;
1114         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1115         r->val = (ytreg & 0x7f);
1116 }
1117
1118 static inline void
1119 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1120                      struct unw_state_record *sr)
1121 {
1122         struct unw_reg_info *r;
1123
1124         if (!desc_is_active(qp, t, sr))
1125                 return;
1126
1127         r = sr->curr.reg + decode_abreg(abreg, 1);
1128         r->where = UNW_WHERE_PSPREL;
1129         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1130         r->val = 0x10 - 4*pspoff;
1131 }
1132
1133 static inline void
1134 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1135                        struct unw_state_record *sr)
1136 {
1137         struct unw_reg_info *r;
1138
1139         if (!desc_is_active(qp, t, sr))
1140                 return;
1141
1142         r = sr->curr.reg + decode_abreg(abreg, 1);
1143         r->where = UNW_WHERE_SPREL;
1144         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1145         r->val = 4*spoff;
1146 }
1147
1148 #define UNW_DEC_BAD_CODE(code)                  printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1149                                                        code);
1150
1151 /*
1152  * region headers:
1153  */
1154 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg)     desc_prologue(0,r,m,gr,arg)
1155 #define UNW_DEC_PROLOGUE(fmt,b,r,arg)           desc_prologue(b,r,0,32,arg)
1156 /*
1157  * prologue descriptors:
1158  */
1159 #define UNW_DEC_ABI(fmt,a,c,arg)                desc_abi(a,c,arg)
1160 #define UNW_DEC_BR_GR(fmt,b,g,arg)              desc_br_gr(b,g,arg)
1161 #define UNW_DEC_BR_MEM(fmt,b,arg)               desc_br_mem(b,arg)
1162 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg)           desc_frgr_mem(g,f,arg)
1163 #define UNW_DEC_FR_MEM(fmt,f,arg)               desc_fr_mem(f,arg)
1164 #define UNW_DEC_GR_GR(fmt,m,g,arg)              desc_gr_gr(m,g,arg)
1165 #define UNW_DEC_GR_MEM(fmt,m,arg)               desc_gr_mem(m,arg)
1166 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg)        desc_mem_stack_f(t,s,arg)
1167 #define UNW_DEC_MEM_STACK_V(fmt,t,arg)          desc_mem_stack_v(t,arg)
1168 #define UNW_DEC_REG_GR(fmt,r,d,arg)             desc_reg_gr(r,d,arg)
1169 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg)         desc_reg_psprel(r,o,arg)
1170 #define UNW_DEC_REG_SPREL(fmt,r,o,arg)          desc_reg_sprel(r,o,arg)
1171 #define UNW_DEC_REG_WHEN(fmt,r,t,arg)           desc_reg_when(r,t,arg)
1172 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg)      desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1173 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg)     desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1174 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg)           desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1175 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg)       desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1176 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg)        desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1177 #define UNW_DEC_RP_BR(fmt,d,arg)                desc_rp_br(d,arg)
1178 #define UNW_DEC_SPILL_BASE(fmt,o,arg)           desc_spill_base(o,arg)
1179 #define UNW_DEC_SPILL_MASK(fmt,m,arg)           (m = desc_spill_mask(m,arg))
1180 /*
1181  * body descriptors:
1182  */
1183 #define UNW_DEC_EPILOGUE(fmt,t,c,arg)           desc_epilogue(t,c,arg)
1184 #define UNW_DEC_COPY_STATE(fmt,l,arg)           desc_copy_state(l,arg)
1185 #define UNW_DEC_LABEL_STATE(fmt,l,arg)          desc_label_state(l,arg)
1186 /*
1187  * general unwind descriptors:
1188  */
1189 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg)    desc_spill_reg_p(p,t,a,x,y,arg)
1190 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg)        desc_spill_reg_p(0,t,a,x,y,arg)
1191 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg)   desc_spill_psprel_p(p,t,a,o,arg)
1192 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg)       desc_spill_psprel_p(0,t,a,o,arg)
1193 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg)    desc_spill_sprel_p(p,t,a,o,arg)
1194 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg)        desc_spill_sprel_p(0,t,a,o,arg)
1195 #define UNW_DEC_RESTORE_P(f,p,t,a,arg)          desc_restore_p(p,t,a,arg)
1196 #define UNW_DEC_RESTORE(f,t,a,arg)              desc_restore_p(0,t,a,arg)
1197
1198 #include "unwind_decoder.c"
1199
1200 \f
1201 /* Unwind scripts. */
1202
1203 static inline unw_hash_index_t
1204 hash (unsigned long ip)
1205 {
1206 #       define hashmagic        0x9e3779b97f4a7c16UL    /* based on (sqrt(5)/2-1)*2^64 */
1207
1208         return (ip >> 4)*hashmagic >> (64 - UNW_LOG_HASH_SIZE);
1209 #undef hashmagic
1210 }
1211
1212 static inline long
1213 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1214 {
1215         read_lock(&script->lock);
1216         if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1217                 /* keep the read lock... */
1218                 return 1;
1219         read_unlock(&script->lock);
1220         return 0;
1221 }
1222
1223 static inline struct unw_script *
1224 script_lookup (struct unw_frame_info *info)
1225 {
1226         struct unw_script *script = unw.cache + info->hint;
1227         unsigned short index;
1228         unsigned long ip, pr;
1229
1230         if (UNW_DEBUG_ON(0))
1231                 return NULL;    /* Always regenerate scripts in debug mode */
1232
1233         STAT(++unw.stat.cache.lookups);
1234
1235         ip = info->ip;
1236         pr = info->pr;
1237
1238         if (cache_match(script, ip, pr)) {
1239                 STAT(++unw.stat.cache.hinted_hits);
1240                 return script;
1241         }
1242
1243         index = unw.hash[hash(ip)];
1244         if (index >= UNW_CACHE_SIZE)
1245                 return NULL;
1246
1247         script = unw.cache + index;
1248         while (1) {
1249                 if (cache_match(script, ip, pr)) {
1250                         /* update hint; no locking required as single-word writes are atomic */
1251                         STAT(++unw.stat.cache.normal_hits);
1252                         unw.cache[info->prev_script].hint = script - unw.cache;
1253                         return script;
1254                 }
1255                 if (script->coll_chain >= UNW_HASH_SIZE)
1256                         return NULL;
1257                 script = unw.cache + script->coll_chain;
1258                 STAT(++unw.stat.cache.collision_chain_traversals);
1259         }
1260 }
1261
1262 /*
1263  * On returning, a write lock for the SCRIPT is still being held.
1264  */
1265 static inline struct unw_script *
1266 script_new (unsigned long ip)
1267 {
1268         struct unw_script *script, *prev, *tmp;
1269         unw_hash_index_t index;
1270         unsigned short head;
1271
1272         STAT(++unw.stat.script.news);
1273
1274         /*
1275          * Can't (easily) use cmpxchg() here because of ABA problem
1276          * that is intrinsic in cmpxchg()...
1277          */
1278         head = unw.lru_head;
1279         script = unw.cache + head;
1280         unw.lru_head = script->lru_chain;
1281
1282         /*
1283          * We'd deadlock here if we interrupted a thread that is holding a read lock on
1284          * script->lock.  Thus, if the write_trylock() fails, we simply bail out.  The
1285          * alternative would be to disable interrupts whenever we hold a read-lock, but
1286          * that seems silly.
1287          */
1288         if (!write_trylock(&script->lock))
1289                 return NULL;
1290
1291         /* re-insert script at the tail of the LRU chain: */
1292         unw.cache[unw.lru_tail].lru_chain = head;
1293         unw.lru_tail = head;
1294
1295         /* remove the old script from the hash table (if it's there): */
1296         if (script->ip) {
1297                 index = hash(script->ip);
1298                 tmp = unw.cache + unw.hash[index];
1299                 prev = NULL;
1300                 while (1) {
1301                         if (tmp == script) {
1302                                 if (prev)
1303                                         prev->coll_chain = tmp->coll_chain;
1304                                 else
1305                                         unw.hash[index] = tmp->coll_chain;
1306                                 break;
1307                         } else
1308                                 prev = tmp;
1309                         if (tmp->coll_chain >= UNW_CACHE_SIZE)
1310                         /* old script wasn't in the hash-table */
1311                                 break;
1312                         tmp = unw.cache + tmp->coll_chain;
1313                 }
1314         }
1315
1316         /* enter new script in the hash table */
1317         index = hash(ip);
1318         script->coll_chain = unw.hash[index];
1319         unw.hash[index] = script - unw.cache;
1320
1321         script->ip = ip;        /* set new IP while we're holding the locks */
1322
1323         STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1324
1325         script->flags = 0;
1326         script->hint = 0;
1327         script->count = 0;
1328         return script;
1329 }
1330
1331 static void
1332 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1333 {
1334         script->pr_mask = sr->pr_mask;
1335         script->pr_val = sr->pr_val;
1336         /*
1337          * We could down-grade our write-lock on script->lock here but
1338          * the rwlock API doesn't offer atomic lock downgrading, so
1339          * we'll just keep the write-lock and release it later when
1340          * we're done using the script.
1341          */
1342 }
1343
1344 static inline void
1345 script_emit (struct unw_script *script, struct unw_insn insn)
1346 {
1347         if (script->count >= UNW_MAX_SCRIPT_LEN) {
1348                 UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1349                         __FUNCTION__, UNW_MAX_SCRIPT_LEN);
1350                 return;
1351         }
1352         script->insn[script->count++] = insn;
1353 }
1354
1355 static inline void
1356 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1357 {
1358         struct unw_reg_info *r = sr->curr.reg + i;
1359         enum unw_insn_opcode opc;
1360         struct unw_insn insn;
1361         unsigned long val = 0;
1362
1363         switch (r->where) {
1364               case UNW_WHERE_GR:
1365                 if (r->val >= 32) {
1366                         /* register got spilled to a stacked register */
1367                         opc = UNW_INSN_SETNAT_TYPE;
1368                         val = UNW_NAT_REGSTK;
1369                 } else
1370                         /* register got spilled to a scratch register */
1371                         opc = UNW_INSN_SETNAT_MEMSTK;
1372                 break;
1373
1374               case UNW_WHERE_FR:
1375                 opc = UNW_INSN_SETNAT_TYPE;
1376                 val = UNW_NAT_VAL;
1377                 break;
1378
1379               case UNW_WHERE_BR:
1380                 opc = UNW_INSN_SETNAT_TYPE;
1381                 val = UNW_NAT_NONE;
1382                 break;
1383
1384               case UNW_WHERE_PSPREL:
1385               case UNW_WHERE_SPREL:
1386                 opc = UNW_INSN_SETNAT_MEMSTK;
1387                 break;
1388
1389               default:
1390                 UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1391                            __FUNCTION__, r->where);
1392                 return;
1393         }
1394         insn.opc = opc;
1395         insn.dst = unw.preg_index[i];
1396         insn.val = val;
1397         script_emit(script, insn);
1398 }
1399
1400 static void
1401 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1402 {
1403         struct unw_reg_info *r = sr->curr.reg + i;
1404         enum unw_insn_opcode opc;
1405         unsigned long val, rval;
1406         struct unw_insn insn;
1407         long need_nat_info;
1408
1409         if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1410                 return;
1411
1412         opc = UNW_INSN_MOVE;
1413         val = rval = r->val;
1414         need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1415
1416         switch (r->where) {
1417               case UNW_WHERE_GR:
1418                 if (rval >= 32) {
1419                         opc = UNW_INSN_MOVE_STACKED;
1420                         val = rval - 32;
1421                 } else if (rval >= 4 && rval <= 7) {
1422                         if (need_nat_info) {
1423                                 opc = UNW_INSN_MOVE2;
1424                                 need_nat_info = 0;
1425                         }
1426                         val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1427                 } else if (rval == 0) {
1428                         opc = UNW_INSN_MOVE_CONST;
1429                         val = 0;
1430                 } else {
1431                         /* register got spilled to a scratch register */
1432                         opc = UNW_INSN_MOVE_SCRATCH;
1433                         val = pt_regs_off(rval);
1434                 }
1435                 break;
1436
1437               case UNW_WHERE_FR:
1438                 if (rval <= 5)
1439                         val = unw.preg_index[UNW_REG_F2  + (rval -  2)];
1440                 else if (rval >= 16 && rval <= 31)
1441                         val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1442                 else {
1443                         opc = UNW_INSN_MOVE_SCRATCH;
1444                         if (rval <= 11)
1445                                 val = offsetof(struct pt_regs, f6) + 16*(rval - 6);
1446                         else
1447                                 UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1448                                            __FUNCTION__, rval);
1449                 }
1450                 break;
1451
1452               case UNW_WHERE_BR:
1453                 if (rval >= 1 && rval <= 5)
1454                         val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1455                 else {
1456                         opc = UNW_INSN_MOVE_SCRATCH;
1457                         if (rval == 0)
1458                                 val = offsetof(struct pt_regs, b0);
1459                         else if (rval == 6)
1460                                 val = offsetof(struct pt_regs, b6);
1461                         else
1462                                 val = offsetof(struct pt_regs, b7);
1463                 }
1464                 break;
1465
1466               case UNW_WHERE_SPREL:
1467                 opc = UNW_INSN_ADD_SP;
1468                 break;
1469
1470               case UNW_WHERE_PSPREL:
1471                 opc = UNW_INSN_ADD_PSP;
1472                 break;
1473
1474               default:
1475                 UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1476                            __FUNCTION__, i, r->where);
1477                 break;
1478         }
1479         insn.opc = opc;
1480         insn.dst = unw.preg_index[i];
1481         insn.val = val;
1482         script_emit(script, insn);
1483         if (need_nat_info)
1484                 emit_nat_info(sr, i, script);
1485
1486         if (i == UNW_REG_PSP) {
1487                 /*
1488                  * info->psp must contain the _value_ of the previous
1489                  * sp, not it's save location.  We get this by
1490                  * dereferencing the value we just stored in
1491                  * info->psp:
1492                  */
1493                 insn.opc = UNW_INSN_LOAD;
1494                 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1495                 script_emit(script, insn);
1496         }
1497 }
1498
1499 static inline const struct unw_table_entry *
1500 lookup (struct unw_table *table, unsigned long rel_ip)
1501 {
1502         const struct unw_table_entry *e = NULL;
1503         unsigned long lo, hi, mid;
1504
1505         /* do a binary search for right entry: */
1506         for (lo = 0, hi = table->length; lo < hi; ) {
1507                 mid = (lo + hi) / 2;
1508                 e = &table->array[mid];
1509                 if (rel_ip < e->start_offset)
1510                         hi = mid;
1511                 else if (rel_ip >= e->end_offset)
1512                         lo = mid + 1;
1513                 else
1514                         break;
1515         }
1516         if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1517                 return NULL;
1518         return e;
1519 }
1520
1521 /*
1522  * Build an unwind script that unwinds from state OLD_STATE to the
1523  * entrypoint of the function that called OLD_STATE.
1524  */
1525 static inline struct unw_script *
1526 build_script (struct unw_frame_info *info)
1527 {
1528         const struct unw_table_entry *e = NULL;
1529         struct unw_script *script = NULL;
1530         struct unw_labeled_state *ls, *next;
1531         unsigned long ip = info->ip;
1532         struct unw_state_record sr;
1533         struct unw_table *table;
1534         struct unw_reg_info *r;
1535         struct unw_insn insn;
1536         u8 *dp, *desc_end;
1537         u64 hdr;
1538         int i;
1539         STAT(unsigned long start, parse_start;)
1540
1541         STAT(++unw.stat.script.builds; start = ia64_get_itc());
1542
1543         /* build state record */
1544         memset(&sr, 0, sizeof(sr));
1545         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1546                 r->when = UNW_WHEN_NEVER;
1547         sr.pr_val = info->pr;
1548
1549         UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __FUNCTION__, ip);
1550         script = script_new(ip);
1551         if (!script) {
1552                 UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n",  __FUNCTION__);
1553                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1554                 return NULL;
1555         }
1556         unw.cache[info->prev_script].hint = script - unw.cache;
1557
1558         /* search the kernels and the modules' unwind tables for IP: */
1559
1560         STAT(parse_start = ia64_get_itc());
1561
1562         for (table = unw.tables; table; table = table->next) {
1563                 if (ip >= table->start && ip < table->end) {
1564                         e = lookup(table, ip - table->segment_base);
1565                         break;
1566                 }
1567         }
1568         if (!e) {
1569                 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs)  */
1570                 UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1571                         __FUNCTION__, ip, unw.cache[info->prev_script].ip);
1572                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1573                 sr.curr.reg[UNW_REG_RP].when = -1;
1574                 sr.curr.reg[UNW_REG_RP].val = 0;
1575                 compile_reg(&sr, UNW_REG_RP, script);
1576                 script_finalize(script, &sr);
1577                 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1578                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1579                 return script;
1580         }
1581
1582         sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1583                           + (ip & 0xfUL));
1584         hdr = *(u64 *) (table->segment_base + e->info_offset);
1585         dp =   (u8 *)  (table->segment_base + e->info_offset + 8);
1586         desc_end = dp + 8*UNW_LENGTH(hdr);
1587
1588         while (!sr.done && dp < desc_end)
1589                 dp = unw_decode(dp, sr.in_body, &sr);
1590
1591         if (sr.when_target > sr.epilogue_start) {
1592                 /*
1593                  * sp has been restored and all values on the memory stack below
1594                  * psp also have been restored.
1595                  */
1596                 sr.curr.reg[UNW_REG_PSP].val = 0;
1597                 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1598                 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1599                 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1600                         if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1601                             || r->where == UNW_WHERE_SPREL)
1602                         {
1603                                 r->val = 0;
1604                                 r->where = UNW_WHERE_NONE;
1605                                 r->when = UNW_WHEN_NEVER;
1606                         }
1607         }
1608
1609         script->flags = sr.flags;
1610
1611         /*
1612          * If RP did't get saved, generate entry for the return link
1613          * register.
1614          */
1615         if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1616                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1617                 sr.curr.reg[UNW_REG_RP].when = -1;
1618                 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1619                 UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1620                            __FUNCTION__, ip, sr.curr.reg[UNW_REG_RP].where,
1621                            sr.curr.reg[UNW_REG_RP].val);
1622         }
1623
1624 #ifdef UNW_DEBUG
1625         UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1626                 __FUNCTION__, table->segment_base + e->start_offset, sr.when_target);
1627         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1628                 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1629                         UNW_DPRINT(1, "  %s <- ", unw.preg_name[r - sr.curr.reg]);
1630                         switch (r->where) {
1631                               case UNW_WHERE_GR:     UNW_DPRINT(1, "r%lu", r->val); break;
1632                               case UNW_WHERE_FR:     UNW_DPRINT(1, "f%lu", r->val); break;
1633                               case UNW_WHERE_BR:     UNW_DPRINT(1, "b%lu", r->val); break;
1634                               case UNW_WHERE_SPREL:  UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1635                               case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1636                               case UNW_WHERE_NONE:
1637                                 UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1638                                 break;
1639
1640                               default:
1641                                 UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1642                                 break;
1643                         }
1644                         UNW_DPRINT(1, "\t\t%d\n", r->when);
1645                 }
1646         }
1647 #endif
1648
1649         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1650
1651         /* translate state record into unwinder instructions: */
1652
1653         /*
1654          * First, set psp if we're dealing with a fixed-size frame;
1655          * subsequent instructions may depend on this value.
1656          */
1657         if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1658             && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1659             && sr.curr.reg[UNW_REG_PSP].val != 0) {
1660                 /* new psp is sp plus frame size */
1661                 insn.opc = UNW_INSN_ADD;
1662                 insn.dst = offsetof(struct unw_frame_info, psp)/8;
1663                 insn.val = sr.curr.reg[UNW_REG_PSP].val;        /* frame size */
1664                 script_emit(script, insn);
1665         }
1666
1667         /* determine where the primary UNaT is: */
1668         if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1669                 i = UNW_REG_PRI_UNAT_MEM;
1670         else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1671                 i = UNW_REG_PRI_UNAT_GR;
1672         else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1673                 i = UNW_REG_PRI_UNAT_MEM;
1674         else
1675                 i = UNW_REG_PRI_UNAT_GR;
1676
1677         compile_reg(&sr, i, script);
1678
1679         for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1680                 compile_reg(&sr, i, script);
1681
1682         /* free labeled register states & stack: */
1683
1684         STAT(parse_start = ia64_get_itc());
1685         for (ls = sr.labeled_states; ls; ls = next) {
1686                 next = ls->next;
1687                 free_state_stack(&ls->saved_state);
1688                 free_labeled_state(ls);
1689         }
1690         free_state_stack(&sr.curr);
1691         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1692
1693         script_finalize(script, &sr);
1694         STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1695         return script;
1696 }
1697
1698 /*
1699  * Apply the unwinding actions represented by OPS and update SR to
1700  * reflect the state that existed upon entry to the function that this
1701  * unwinder represents.
1702  */
1703 static inline void
1704 run_script (struct unw_script *script, struct unw_frame_info *state)
1705 {
1706         struct unw_insn *ip, *limit, next_insn;
1707         unsigned long opc, dst, val, off;
1708         unsigned long *s = (unsigned long *) state;
1709         STAT(unsigned long start;)
1710
1711         STAT(++unw.stat.script.runs; start = ia64_get_itc());
1712         state->flags = script->flags;
1713         ip = script->insn;
1714         limit = script->insn + script->count;
1715         next_insn = *ip;
1716
1717         while (ip++ < limit) {
1718                 opc = next_insn.opc;
1719                 dst = next_insn.dst;
1720                 val = next_insn.val;
1721                 next_insn = *ip;
1722
1723           redo:
1724                 switch (opc) {
1725                       case UNW_INSN_ADD:
1726                         s[dst] += val;
1727                         break;
1728
1729                       case UNW_INSN_MOVE2:
1730                         if (!s[val])
1731                                 goto lazy_init;
1732                         s[dst+1] = s[val+1];
1733                         s[dst] = s[val];
1734                         break;
1735
1736                       case UNW_INSN_MOVE:
1737                         if (!s[val])
1738                                 goto lazy_init;
1739                         s[dst] = s[val];
1740                         break;
1741
1742                       case UNW_INSN_MOVE_SCRATCH:
1743                         if (state->pt) {
1744                                 s[dst] = (unsigned long) get_scratch_regs(state) + val;
1745                         } else {
1746                                 s[dst] = 0;
1747                                 UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1748                                            __FUNCTION__, dst, val);
1749                         }
1750                         break;
1751
1752                       case UNW_INSN_MOVE_CONST:
1753                         if (val == 0)
1754                                 s[dst] = (unsigned long) &unw.r0;
1755                         else {
1756                                 s[dst] = 0;
1757                                 UNW_DPRINT(0, "unwind.%s: UNW_INSN_MOVE_CONST bad val=%ld\n",
1758                                            __FUNCTION__, val);
1759                         }
1760                         break;
1761
1762
1763                       case UNW_INSN_MOVE_STACKED:
1764                         s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1765                                                                     val);
1766                         break;
1767
1768                       case UNW_INSN_ADD_PSP:
1769                         s[dst] = state->psp + val;
1770                         break;
1771
1772                       case UNW_INSN_ADD_SP:
1773                         s[dst] = state->sp + val;
1774                         break;
1775
1776                       case UNW_INSN_SETNAT_MEMSTK:
1777                         if (!state->pri_unat_loc)
1778                                 state->pri_unat_loc = &state->sw->caller_unat;
1779                         /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1780                         s[dst+1] = ((unsigned long) state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1781                         break;
1782
1783                       case UNW_INSN_SETNAT_TYPE:
1784                         s[dst+1] = val;
1785                         break;
1786
1787                       case UNW_INSN_LOAD:
1788 #ifdef UNW_DEBUG
1789                         if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1790                             || s[val] < TASK_SIZE)
1791                         {
1792                                 UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1793                                            __FUNCTION__, s[val]);
1794                                 break;
1795                         }
1796 #endif
1797                         s[dst] = *(unsigned long *) s[val];
1798                         break;
1799                 }
1800         }
1801         STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1802         return;
1803
1804   lazy_init:
1805         off = unw.sw_off[val];
1806         s[val] = (unsigned long) state->sw + off;
1807         if (off >= offsetof(struct switch_stack, r4) && off <= offsetof(struct switch_stack, r7))
1808                 /*
1809                  * We're initializing a general register: init NaT info, too.  Note that
1810                  * the offset is a multiple of 8 which gives us the 3 bits needed for
1811                  * the type field.
1812                  */
1813                 s[val+1] = (offsetof(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1814         goto redo;
1815 }
1816
1817 static int
1818 find_save_locs (struct unw_frame_info *info)
1819 {
1820         int have_write_lock = 0;
1821         struct unw_script *scr;
1822         unsigned long flags = 0;
1823
1824         if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1825                 /* don't let obviously bad addresses pollute the cache */
1826                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1827                 UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __FUNCTION__, info->ip);
1828                 info->rp_loc = NULL;
1829                 return -1;
1830         }
1831
1832         scr = script_lookup(info);
1833         if (!scr) {
1834                 spin_lock_irqsave(&unw.lock, flags);
1835                 scr = build_script(info);
1836                 if (!scr) {
1837                         spin_unlock_irqrestore(&unw.lock, flags);
1838                         UNW_DPRINT(0,
1839                                    "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1840                                    __FUNCTION__, info->ip);
1841                         return -1;
1842                 }
1843                 have_write_lock = 1;
1844         }
1845         info->hint = scr->hint;
1846         info->prev_script = scr - unw.cache;
1847
1848         run_script(scr, info);
1849
1850         if (have_write_lock) {
1851                 write_unlock(&scr->lock);
1852                 spin_unlock_irqrestore(&unw.lock, flags);
1853         } else
1854                 read_unlock(&scr->lock);
1855         return 0;
1856 }
1857
1858 int
1859 unw_unwind (struct unw_frame_info *info)
1860 {
1861         unsigned long prev_ip, prev_sp, prev_bsp;
1862         unsigned long ip, pr, num_regs;
1863         STAT(unsigned long start, flags;)
1864         int retval;
1865
1866         STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1867
1868         prev_ip = info->ip;
1869         prev_sp = info->sp;
1870         prev_bsp = info->bsp;
1871
1872         /* restore the ip */
1873         if (!info->rp_loc) {
1874                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1875                 UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1876                            __FUNCTION__, info->ip);
1877                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1878                 return -1;
1879         }
1880         ip = info->ip = *info->rp_loc;
1881         if (ip < GATE_ADDR) {
1882                 UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __FUNCTION__, ip);
1883                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1884                 return -1;
1885         }
1886
1887         /* restore the cfm: */
1888         if (!info->pfs_loc) {
1889                 UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __FUNCTION__);
1890                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1891                 return -1;
1892         }
1893         info->cfm_loc = info->pfs_loc;
1894
1895         /* restore the bsp: */
1896         pr = info->pr;
1897         num_regs = 0;
1898         if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1899                 info->pt = info->sp + 16;
1900                 if ((pr & (1UL << PRED_NON_SYSCALL)) != 0)
1901                         num_regs = *info->cfm_loc & 0x7f;               /* size of frame */
1902                 info->pfs_loc =
1903                         (unsigned long *) (info->pt + offsetof(struct pt_regs, ar_pfs));
1904                 UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __FUNCTION__, info->pt);
1905         } else
1906                 num_regs = (*info->cfm_loc >> 7) & 0x7f;        /* size of locals */
1907         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1908         if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1909                 UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1910                         __FUNCTION__, info->bsp, info->regstk.limit, info->regstk.top);
1911                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1912                 return -1;
1913         }
1914
1915         /* restore the sp: */
1916         info->sp = info->psp;
1917         if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1918                 UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1919                         __FUNCTION__, info->sp, info->memstk.top, info->memstk.limit);
1920                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1921                 return -1;
1922         }
1923
1924         if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1925                 UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1926                            __FUNCTION__, ip);
1927                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1928                 return -1;
1929         }
1930
1931         /* as we unwind, the saved ar.unat becomes the primary unat: */
1932         info->pri_unat_loc = info->unat_loc;
1933
1934         /* finally, restore the predicates: */
1935         unw_get_pr(info, &info->pr);
1936
1937         retval = find_save_locs(info);
1938         STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1939         return retval;
1940 }
1941 EXPORT_SYMBOL(unw_unwind);
1942
1943 int
1944 unw_unwind_to_user (struct unw_frame_info *info)
1945 {
1946         unsigned long ip, sp, pr = 0;
1947
1948         while (unw_unwind(info) >= 0) {
1949                 unw_get_sp(info, &sp);
1950                 if ((long)((unsigned long)info->task + IA64_STK_OFFSET - sp)
1951                     < IA64_PT_REGS_SIZE) {
1952                         UNW_DPRINT(0, "unwind.%s: ran off the top of the kernel stack\n",
1953                                    __FUNCTION__);
1954                         break;
1955                 }
1956                 if (unw_is_intr_frame(info) &&
1957                     (pr & (1UL << PRED_USER_STACK)))
1958                         return 0;
1959                 if (unw_get_pr (info, &pr) < 0) {
1960                         unw_get_rp(info, &ip);
1961                         UNW_DPRINT(0, "unwind.%s: failed to read "
1962                                    "predicate register (ip=0x%lx)\n",
1963                                 __FUNCTION__, ip);
1964                         return -1;
1965                 }
1966         }
1967         unw_get_ip(info, &ip);
1968         UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n",
1969                    __FUNCTION__, ip);
1970         return -1;
1971 }
1972 EXPORT_SYMBOL(unw_unwind_to_user);
1973
1974 static void
1975 init_frame_info (struct unw_frame_info *info, struct task_struct *t,
1976                  struct switch_stack *sw, unsigned long stktop)
1977 {
1978         unsigned long rbslimit, rbstop, stklimit;
1979         STAT(unsigned long start, flags;)
1980
1981         STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
1982
1983         /*
1984          * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
1985          * don't want to do that because it would be slow as each preserved register would
1986          * have to be processed.  Instead, what we do here is zero out the frame info and
1987          * start the unwind process at the function that created the switch_stack frame.
1988          * When a preserved value in switch_stack needs to be accessed, run_script() will
1989          * initialize the appropriate pointer on demand.
1990          */
1991         memset(info, 0, sizeof(*info));
1992
1993         rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
1994         rbstop   = sw->ar_bspstore;
1995         if (rbstop - (unsigned long) t >= IA64_STK_OFFSET)
1996                 rbstop = rbslimit;
1997
1998         stklimit = (unsigned long) t + IA64_STK_OFFSET;
1999         if (stktop <= rbstop)
2000                 stktop = rbstop;
2001
2002         info->regstk.limit = rbslimit;
2003         info->regstk.top   = rbstop;
2004         info->memstk.limit = stklimit;
2005         info->memstk.top   = stktop;
2006         info->task = t;
2007         info->sw  = sw;
2008         info->sp = info->psp = stktop;
2009         info->pr = sw->pr;
2010         UNW_DPRINT(3, "unwind.%s:\n"
2011                    "  task   0x%lx\n"
2012                    "  rbs = [0x%lx-0x%lx)\n"
2013                    "  stk = [0x%lx-0x%lx)\n"
2014                    "  pr     0x%lx\n"
2015                    "  sw     0x%lx\n"
2016                    "  sp     0x%lx\n",
2017                    __FUNCTION__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
2018                    info->pr, (unsigned long) info->sw, info->sp);
2019         STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
2020 }
2021
2022 void
2023 unw_init_from_interruption (struct unw_frame_info *info, struct task_struct *t,
2024                             struct pt_regs *pt, struct switch_stack *sw)
2025 {
2026         unsigned long sof;
2027
2028         init_frame_info(info, t, sw, pt->r12);
2029         info->cfm_loc = &pt->cr_ifs;
2030         info->unat_loc = &pt->ar_unat;
2031         info->pfs_loc = &pt->ar_pfs;
2032         sof = *info->cfm_loc & 0x7f;
2033         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sof);
2034         info->ip = pt->cr_iip + ia64_psr(pt)->ri;
2035         info->pt = (unsigned long) pt;
2036         UNW_DPRINT(3, "unwind.%s:\n"
2037                    "  bsp    0x%lx\n"
2038                    "  sof    0x%lx\n"
2039                    "  ip     0x%lx\n",
2040                    __FUNCTION__, info->bsp, sof, info->ip);
2041         find_save_locs(info);
2042 }
2043
2044 void
2045 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
2046 {
2047         unsigned long sol;
2048
2049         init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
2050         info->cfm_loc = &sw->ar_pfs;
2051         sol = (*info->cfm_loc >> 7) & 0x7f;
2052         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
2053         info->ip = sw->b0;
2054         UNW_DPRINT(3, "unwind.%s:\n"
2055                    "  bsp    0x%lx\n"
2056                    "  sol    0x%lx\n"
2057                    "  ip     0x%lx\n",
2058                    __FUNCTION__, info->bsp, sol, info->ip);
2059         find_save_locs(info);
2060 }
2061
2062 EXPORT_SYMBOL(unw_init_frame_info);
2063
2064 void
2065 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
2066 {
2067         struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
2068
2069         UNW_DPRINT(1, "unwind.%s\n", __FUNCTION__);
2070         unw_init_frame_info(info, t, sw);
2071 }
2072 EXPORT_SYMBOL(unw_init_from_blocked_task);
2073
2074 static void
2075 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
2076                    unsigned long gp, const void *table_start, const void *table_end)
2077 {
2078         const struct unw_table_entry *start = table_start, *end = table_end;
2079
2080         table->name = name;
2081         table->segment_base = segment_base;
2082         table->gp = gp;
2083         table->start = segment_base + start[0].start_offset;
2084         table->end = segment_base + end[-1].end_offset;
2085         table->array = start;
2086         table->length = end - start;
2087 }
2088
2089 void *
2090 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2091                       const void *table_start, const void *table_end)
2092 {
2093         const struct unw_table_entry *start = table_start, *end = table_end;
2094         struct unw_table *table;
2095         unsigned long flags;
2096
2097         if (end - start <= 0) {
2098                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2099                            __FUNCTION__);
2100                 return NULL;
2101         }
2102
2103         table = kmalloc(sizeof(*table), GFP_USER);
2104         if (!table)
2105                 return NULL;
2106
2107         init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2108
2109         spin_lock_irqsave(&unw.lock, flags);
2110         {
2111                 /* keep kernel unwind table at the front (it's searched most commonly): */
2112                 table->next = unw.tables->next;
2113                 unw.tables->next = table;
2114         }
2115         spin_unlock_irqrestore(&unw.lock, flags);
2116
2117         return table;
2118 }
2119
2120 void
2121 unw_remove_unwind_table (void *handle)
2122 {
2123         struct unw_table *table, *prev;
2124         struct unw_script *tmp;
2125         unsigned long flags;
2126         long index;
2127
2128         if (!handle) {
2129                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2130                            __FUNCTION__);
2131                 return;
2132         }
2133
2134         table = handle;
2135         if (table == &unw.kernel_table) {
2136                 UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2137                            "no-can-do!\n", __FUNCTION__);
2138                 return;
2139         }
2140
2141         spin_lock_irqsave(&unw.lock, flags);
2142         {
2143                 /* first, delete the table: */
2144
2145                 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2146                         if (prev->next == table)
2147                                 break;
2148                 if (!prev) {
2149                         UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2150                                    __FUNCTION__, (void *) table);
2151                         spin_unlock_irqrestore(&unw.lock, flags);
2152                         return;
2153                 }
2154                 prev->next = table->next;
2155         }
2156         spin_unlock_irqrestore(&unw.lock, flags);
2157
2158         /* next, remove hash table entries for this table */
2159
2160         for (index = 0; index <= UNW_HASH_SIZE; ++index) {
2161                 tmp = unw.cache + unw.hash[index];
2162                 if (unw.hash[index] >= UNW_CACHE_SIZE
2163                     || tmp->ip < table->start || tmp->ip >= table->end)
2164                         continue;
2165
2166                 write_lock(&tmp->lock);
2167                 {
2168                         if (tmp->ip >= table->start && tmp->ip < table->end) {
2169                                 unw.hash[index] = tmp->coll_chain;
2170                                 tmp->ip = 0;
2171                         }
2172                 }
2173                 write_unlock(&tmp->lock);
2174         }
2175
2176         kfree(table);
2177 }
2178
2179 static int __init
2180 create_gate_table (void)
2181 {
2182         const struct unw_table_entry *entry, *start, *end;
2183         unsigned long *lp, segbase = GATE_ADDR;
2184         size_t info_size, size;
2185         char *info;
2186         Elf64_Phdr *punw = NULL, *phdr = (Elf64_Phdr *) (GATE_ADDR + GATE_EHDR->e_phoff);
2187         int i;
2188
2189         for (i = 0; i < GATE_EHDR->e_phnum; ++i, ++phdr)
2190                 if (phdr->p_type == PT_IA_64_UNWIND) {
2191                         punw = phdr;
2192                         break;
2193                 }
2194
2195         if (!punw) {
2196                 printk("%s: failed to find gate DSO's unwind table!\n", __FUNCTION__);
2197                 return 0;
2198         }
2199
2200         start = (const struct unw_table_entry *) punw->p_vaddr;
2201         end = (struct unw_table_entry *) ((char *) start + punw->p_memsz);
2202         size  = 0;
2203
2204         unw_add_unwind_table("linux-gate.so", segbase, 0, start, end);
2205
2206         for (entry = start; entry < end; ++entry)
2207                 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2208         size += 8;      /* reserve space for "end of table" marker */
2209
2210         unw.gate_table = kmalloc(size, GFP_KERNEL);
2211         if (!unw.gate_table) {
2212                 unw.gate_table_size = 0;
2213                 printk(KERN_ERR "%s: unable to create unwind data for gate page!\n", __FUNCTION__);
2214                 return 0;
2215         }
2216         unw.gate_table_size = size;
2217
2218         lp = unw.gate_table;
2219         info = (char *) unw.gate_table + size;
2220
2221         for (entry = start; entry < end; ++entry, lp += 3) {
2222                 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2223                 info -= info_size;
2224                 memcpy(info, (char *) segbase + entry->info_offset, info_size);
2225
2226                 lp[0] = segbase + entry->start_offset;          /* start */
2227                 lp[1] = segbase + entry->end_offset;            /* end */
2228                 lp[2] = info - (char *) unw.gate_table;         /* info */
2229         }
2230         *lp = 0;        /* end-of-table marker */
2231         return 0;
2232 }
2233
2234 __initcall(create_gate_table);
2235
2236 void __init
2237 unw_init (void)
2238 {
2239         extern char __gp[];
2240         extern void unw_hash_index_t_is_too_narrow (void);
2241         long i, off;
2242
2243         if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2244                 unw_hash_index_t_is_too_narrow();
2245
2246         unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(CALLER_UNAT);
2247         unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2248         unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_PFS);
2249         unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2250         unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(CALLER_UNAT);
2251         unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2252         unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2253         unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2254         for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2255                 unw.sw_off[unw.preg_index[i]] = off;
2256         for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2257                 unw.sw_off[unw.preg_index[i]] = off;
2258         for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2259                 unw.sw_off[unw.preg_index[i]] = off;
2260         for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2261                 unw.sw_off[unw.preg_index[i]] = off;
2262
2263         for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2264                 if (i > 0)
2265                         unw.cache[i].lru_chain = (i - 1);
2266                 unw.cache[i].coll_chain = -1;
2267                 rwlock_init(&unw.cache[i].lock);
2268         }
2269         unw.lru_head = UNW_CACHE_SIZE - 1;
2270         unw.lru_tail = 0;
2271
2272         init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) __gp,
2273                           __start_unwind, __end_unwind);
2274 }
2275
2276 /*
2277  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2278  *
2279  *      This system call has been deprecated.  The new and improved way to get
2280  *      at the kernel's unwind info is via the gate DSO.  The address of the
2281  *      ELF header for this DSO is passed to user-level via AT_SYSINFO_EHDR.
2282  *
2283  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2284  *
2285  * This system call copies the unwind data into the buffer pointed to by BUF and returns
2286  * the size of the unwind data.  If BUF_SIZE is smaller than the size of the unwind data
2287  * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2288  * unwind data.
2289  *
2290  * The first portion of the unwind data contains an unwind table and rest contains the
2291  * associated unwind info (in no particular order).  The unwind table consists of a table
2292  * of entries of the form:
2293  *
2294  *      u64 start;      (64-bit address of start of function)
2295  *      u64 end;        (64-bit address of start of function)
2296  *      u64 info;       (BUF-relative offset to unwind info)
2297  *
2298  * The end of the unwind table is indicated by an entry with a START address of zero.
2299  *
2300  * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2301  * on the format of the unwind info.
2302  *
2303  * ERRORS
2304  *      EFAULT  BUF points outside your accessible address space.
2305  */
2306 asmlinkage long
2307 sys_getunwind (void __user *buf, size_t buf_size)
2308 {
2309         if (buf && buf_size >= unw.gate_table_size)
2310                 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2311                         return -EFAULT;
2312         return unw.gate_table_size;
2313 }