arch/arm/kernel/perf_event_xscale.c

   1 /*
   2  * ARMv5 [xscale] Performance counter handling code.
   3  *
   4  * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com>
   5  *
   6  * Based on the previous xscale OProfile code.
   7  *
   8  * There are two variants of the xscale PMU that we support:
   9  *      - xscale1pmu: 2 event counters and a cycle counter
  10  *      - xscale2pmu: 4 event counters and a cycle counter
  11  * The two variants share event definitions, but have different
  12  * PMU structures.
  13  */
  14
  15 #ifdef CONFIG_CPU_XSCALE
  16 enum xscale_perf_types {
  17         XSCALE_PERFCTR_ICACHE_MISS              = 0x00,
  18         XSCALE_PERFCTR_ICACHE_NO_DELIVER        = 0x01,
  19         XSCALE_PERFCTR_DATA_STALL               = 0x02,
  20         XSCALE_PERFCTR_ITLB_MISS                = 0x03,
  21         XSCALE_PERFCTR_DTLB_MISS                = 0x04,
  22         XSCALE_PERFCTR_BRANCH                   = 0x05,
  23         XSCALE_PERFCTR_BRANCH_MISS              = 0x06,
  24         XSCALE_PERFCTR_INSTRUCTION              = 0x07,
  25         XSCALE_PERFCTR_DCACHE_FULL_STALL        = 0x08,
  26         XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09,
  27         XSCALE_PERFCTR_DCACHE_ACCESS            = 0x0A,
  28         XSCALE_PERFCTR_DCACHE_MISS              = 0x0B,
  29         XSCALE_PERFCTR_DCACHE_WRITE_BACK        = 0x0C,
  30         XSCALE_PERFCTR_PC_CHANGED               = 0x0D,
  31         XSCALE_PERFCTR_BCU_REQUEST              = 0x10,
  32         XSCALE_PERFCTR_BCU_FULL                 = 0x11,
  33         XSCALE_PERFCTR_BCU_DRAIN                = 0x12,
  34         XSCALE_PERFCTR_BCU_ECC_NO_ELOG          = 0x14,
  35         XSCALE_PERFCTR_BCU_1_BIT_ERR            = 0x15,
  36         XSCALE_PERFCTR_RMW                      = 0x16,
  37         /* XSCALE_PERFCTR_CCNT is not hardware defined */
  38         XSCALE_PERFCTR_CCNT                     = 0xFE,
  39         XSCALE_PERFCTR_UNUSED                   = 0xFF,
  40 };
  41
  42 enum xscale_counters {
  43         XSCALE_CYCLE_COUNTER    = 0,
  44         XSCALE_COUNTER0,
  45         XSCALE_COUNTER1,
  46         XSCALE_COUNTER2,
  47         XSCALE_COUNTER3,
  48 };
  49
  50 static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = {
  51         [PERF_COUNT_HW_CPU_CYCLES]          = XSCALE_PERFCTR_CCNT,
  52         [PERF_COUNT_HW_INSTRUCTIONS]        = XSCALE_PERFCTR_INSTRUCTION,
  53         [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
  54         [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
  55         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH,
  56         [PERF_COUNT_HW_BRANCH_MISSES]       = XSCALE_PERFCTR_BRANCH_MISS,
  57         [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
  58 };
  59
  60 static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  61                                            [PERF_COUNT_HW_CACHE_OP_MAX]
  62                                            [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  63         [C(L1D)] = {
  64                 [C(OP_READ)] = {
  65                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
  66                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
  67                 },
  68                 [C(OP_WRITE)] = {
  69                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
  70                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
  71                 },
  72                 [C(OP_PREFETCH)] = {
  73                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  74                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  75                 },
  76         },
  77         [C(L1I)] = {
  78                 [C(OP_READ)] = {
  79                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  80                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
  81                 },
  82                 [C(OP_WRITE)] = {
  83                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  84                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
  85                 },
  86                 [C(OP_PREFETCH)] = {
  87                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  88                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  89                 },
  90         },
  91         [C(LL)] = {
  92                 [C(OP_READ)] = {
  93                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  94                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  95                 },
  96                 [C(OP_WRITE)] = {
  97                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  98                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  99                 },
 100                 [C(OP_PREFETCH)] = {
 101                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 102                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 103                 },
 104         },
 105         [C(DTLB)] = {
 106                 [C(OP_READ)] = {
 107                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 108                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
 109                 },
 110                 [C(OP_WRITE)] = {
 111                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 112                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
 113                 },
 114                 [C(OP_PREFETCH)] = {
 115                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 116                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 117                 },
 118         },
 119         [C(ITLB)] = {
 120                 [C(OP_READ)] = {
 121                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 122                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
 123                 },
 124                 [C(OP_WRITE)] = {
 125                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 126                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
 127                 },
 128                 [C(OP_PREFETCH)] = {
 129                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 130                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 131                 },
 132         },
 133         [C(BPU)] = {
 134                 [C(OP_READ)] = {
 135                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 136                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 137                 },
 138                 [C(OP_WRITE)] = {
 139                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 140                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 141                 },
 142                 [C(OP_PREFETCH)] = {
 143                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 144                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 145                 },
 146         },
 147         [C(NODE)] = {
 148                 [C(OP_READ)] = {
 149                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 150                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 151                 },
 152                 [C(OP_WRITE)] = {
 153                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 154                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 155                 },
 156                 [C(OP_PREFETCH)] = {
 157                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 158                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 159                 },
 160         },
 161 };
 162
 163 #define XSCALE_PMU_ENABLE       0x001
 164 #define XSCALE_PMN_RESET        0x002
 165 #define XSCALE_CCNT_RESET       0x004
 166 #define XSCALE_PMU_RESET        (CCNT_RESET | PMN_RESET)
 167 #define XSCALE_PMU_CNT64        0x008
 168
 169 #define XSCALE1_OVERFLOWED_MASK 0x700
 170 #define XSCALE1_CCOUNT_OVERFLOW 0x400
 171 #define XSCALE1_COUNT0_OVERFLOW 0x100
 172 #define XSCALE1_COUNT1_OVERFLOW 0x200
 173 #define XSCALE1_CCOUNT_INT_EN   0x040
 174 #define XSCALE1_COUNT0_INT_EN   0x010
 175 #define XSCALE1_COUNT1_INT_EN   0x020
 176 #define XSCALE1_COUNT0_EVT_SHFT 12
 177 #define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT)
 178 #define XSCALE1_COUNT1_EVT_SHFT 20
 179 #define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT)
 180
 181 static inline u32
 182 xscale1pmu_read_pmnc(void)
 183 {
 184         u32 val;
 185         asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val));
 186         return val;
 187 }
 188
 189 static inline void
 190 xscale1pmu_write_pmnc(u32 val)
 191 {
 192         /* upper 4bits and 7, 11 are write-as-0 */
 193         val &= 0xffff77f;
 194         asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val));
 195 }
 196
 197 static inline int
 198 xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
 199                                         enum xscale_counters counter)
 200 {
 201         int ret = 0;
 202
 203         switch (counter) {
 204         case XSCALE_CYCLE_COUNTER:
 205                 ret = pmnc & XSCALE1_CCOUNT_OVERFLOW;
 206                 break;
 207         case XSCALE_COUNTER0:
 208                 ret = pmnc & XSCALE1_COUNT0_OVERFLOW;
 209                 break;
 210         case XSCALE_COUNTER1:
 211                 ret = pmnc & XSCALE1_COUNT1_OVERFLOW;
 212                 break;
 213         default:
 214                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
 215         }
 216
 217         return ret;
 218 }
 219
 220 static irqreturn_t
 221 xscale1pmu_handle_irq(int irq_num, void *dev)
 222 {
 223         unsigned long pmnc;
 224         struct perf_sample_data data;
 225         struct pmu_hw_events *cpuc;
 226         struct pt_regs *regs;
 227         int idx;
 228
 229         /*
 230          * NOTE: there's an A stepping erratum that states if an overflow
 231          *       bit already exists and another occurs, the previous
 232          *       Overflow bit gets cleared. There's no workaround.
 233          *       Fixed in B stepping or later.
 234          */
 235         pmnc = xscale1pmu_read_pmnc();
 236
 237         /*
 238          * Write the value back to clear the overflow flags. Overflow
 239          * flags remain in pmnc for use below. We also disable the PMU
 240          * while we process the interrupt.
 241          */
 242         xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
 243
 244         if (!(pmnc & XSCALE1_OVERFLOWED_MASK))
 245                 return IRQ_NONE;
 246
 247         regs = get_irq_regs();
 248
 249         perf_sample_data_init(&data, 0);
 250
 251         cpuc = &__get_cpu_var(cpu_hw_events);
 252         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 253                 struct perf_event *event = cpuc->events[idx];
 254                 struct hw_perf_event *hwc;
 255
 256                 if (!event)
 257                         continue;
 258
 259                 if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx))
 260                         continue;
 261
 262                 hwc = &event->hw;
 263                 armpmu_event_update(event, hwc, idx);
 264                 data.period = event->hw.last_period;
 265                 if (!armpmu_event_set_period(event, hwc, idx))
 266                         continue;
 267
 268                 if (perf_event_overflow(event, &data, regs))
 269                         cpu_pmu->disable(hwc, idx);
 270         }
 271
 272         irq_work_run();
 273
 274         /*
 275          * Re-enable the PMU.
 276          */
 277         pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE;
 278         xscale1pmu_write_pmnc(pmnc);
 279
 280         return IRQ_HANDLED;
 281 }
 282
 283 static void
 284 xscale1pmu_enable_event(struct hw_perf_event *hwc, int idx)
 285 {
 286         unsigned long val, mask, evt, flags;
 287         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 288
 289         switch (idx) {
 290         case XSCALE_CYCLE_COUNTER:
 291                 mask = 0;
 292                 evt = XSCALE1_CCOUNT_INT_EN;
 293                 break;
 294         case XSCALE_COUNTER0:
 295                 mask = XSCALE1_COUNT0_EVT_MASK;
 296                 evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) |
 297                         XSCALE1_COUNT0_INT_EN;
 298                 break;
 299         case XSCALE_COUNTER1:
 300                 mask = XSCALE1_COUNT1_EVT_MASK;
 301                 evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) |
 302                         XSCALE1_COUNT1_INT_EN;
 303                 break;
 304         default:
 305                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 306                 return;
 307         }
 308
 309         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 310         val = xscale1pmu_read_pmnc();
 311         val &= ~mask;
 312         val |= evt;
 313         xscale1pmu_write_pmnc(val);
 314         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 315 }
 316
 317 static void
 318 xscale1pmu_disable_event(struct hw_perf_event *hwc, int idx)
 319 {
 320         unsigned long val, mask, evt, flags;
 321         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 322
 323         switch (idx) {
 324         case XSCALE_CYCLE_COUNTER:
 325                 mask = XSCALE1_CCOUNT_INT_EN;
 326                 evt = 0;
 327                 break;
 328         case XSCALE_COUNTER0:
 329                 mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK;
 330                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT;
 331                 break;
 332         case XSCALE_COUNTER1:
 333                 mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK;
 334                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT;
 335                 break;
 336         default:
 337                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 338                 return;
 339         }
 340
 341         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 342         val = xscale1pmu_read_pmnc();
 343         val &= ~mask;
 344         val |= evt;
 345         xscale1pmu_write_pmnc(val);
 346         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 347 }
 348
 349 static int
 350 xscale1pmu_get_event_idx(struct pmu_hw_events *cpuc,
 351                         struct hw_perf_event *event)
 352 {
 353         if (XSCALE_PERFCTR_CCNT == event->config_base) {
 354                 if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask))
 355                         return -EAGAIN;
 356
 357                 return XSCALE_CYCLE_COUNTER;
 358         } else {
 359                 if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask))
 360                         return XSCALE_COUNTER1;
 361
 362                 if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask))
 363                         return XSCALE_COUNTER0;
 364
 365                 return -EAGAIN;
 366         }
 367 }
 368
 369 static void
 370 xscale1pmu_start(void)
 371 {
 372         unsigned long flags, val;
 373         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 374
 375         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 376         val = xscale1pmu_read_pmnc();
 377         val |= XSCALE_PMU_ENABLE;
 378         xscale1pmu_write_pmnc(val);
 379         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 380 }
 381
 382 static void
 383 xscale1pmu_stop(void)
 384 {
 385         unsigned long flags, val;
 386         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 387
 388         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 389         val = xscale1pmu_read_pmnc();
 390         val &= ~XSCALE_PMU_ENABLE;
 391         xscale1pmu_write_pmnc(val);
 392         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 393 }
 394
 395 static inline u32
 396 xscale1pmu_read_counter(int counter)
 397 {
 398         u32 val = 0;
 399
 400         switch (counter) {
 401         case XSCALE_CYCLE_COUNTER:
 402                 asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val));
 403                 break;
 404         case XSCALE_COUNTER0:
 405                 asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val));
 406                 break;
 407         case XSCALE_COUNTER1:
 408                 asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val));
 409                 break;
 410         }
 411
 412         return val;
 413 }
 414
 415 static inline void
 416 xscale1pmu_write_counter(int counter, u32 val)
 417 {
 418         switch (counter) {
 419         case XSCALE_CYCLE_COUNTER:
 420                 asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val));
 421                 break;
 422         case XSCALE_COUNTER0:
 423                 asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val));
 424                 break;
 425         case XSCALE_COUNTER1:
 426                 asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val));
 427                 break;
 428         }
 429 }
 430
 431 static int xscale_map_event(struct perf_event *event)
 432 {
 433         return map_cpu_event(event, &xscale_perf_map,
 434                                 &xscale_perf_cache_map, 0xFF);
 435 }
 436
 437 static struct arm_pmu xscale1pmu = {
 438         .id             = ARM_PERF_PMU_ID_XSCALE1,
 439         .name           = "xscale1",
 440         .handle_irq     = xscale1pmu_handle_irq,
 441         .enable         = xscale1pmu_enable_event,
 442         .disable        = xscale1pmu_disable_event,
 443         .read_counter   = xscale1pmu_read_counter,
 444         .write_counter  = xscale1pmu_write_counter,
 445         .get_event_idx  = xscale1pmu_get_event_idx,
 446         .start          = xscale1pmu_start,
 447         .stop           = xscale1pmu_stop,
 448         .map_event      = xscale_map_event,
 449         .num_events     = 3,
 450         .max_period     = (1LLU << 32) - 1,
 451 };
 452
 453 static struct arm_pmu *__init xscale1pmu_init(void)
 454 {
 455         return &xscale1pmu;
 456 }
 457
 458 #define XSCALE2_OVERFLOWED_MASK 0x01f
 459 #define XSCALE2_CCOUNT_OVERFLOW 0x001
 460 #define XSCALE2_COUNT0_OVERFLOW 0x002
 461 #define XSCALE2_COUNT1_OVERFLOW 0x004
 462 #define XSCALE2_COUNT2_OVERFLOW 0x008
 463 #define XSCALE2_COUNT3_OVERFLOW 0x010
 464 #define XSCALE2_CCOUNT_INT_EN   0x001
 465 #define XSCALE2_COUNT0_INT_EN   0x002
 466 #define XSCALE2_COUNT1_INT_EN   0x004
 467 #define XSCALE2_COUNT2_INT_EN   0x008
 468 #define XSCALE2_COUNT3_INT_EN   0x010
 469 #define XSCALE2_COUNT0_EVT_SHFT 0
 470 #define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT)
 471 #define XSCALE2_COUNT1_EVT_SHFT 8
 472 #define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT)
 473 #define XSCALE2_COUNT2_EVT_SHFT 16
 474 #define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT)
 475 #define XSCALE2_COUNT3_EVT_SHFT 24
 476 #define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT)
 477
 478 static inline u32
 479 xscale2pmu_read_pmnc(void)
 480 {
 481         u32 val;
 482         asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val));
 483         /* bits 1-2 and 4-23 are read-unpredictable */
 484         return val & 0xff000009;
 485 }
 486
 487 static inline void
 488 xscale2pmu_write_pmnc(u32 val)
 489 {
 490         /* bits 4-23 are write-as-0, 24-31 are write ignored */
 491         val &= 0xf;
 492         asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val));
 493 }
 494
 495 static inline u32
 496 xscale2pmu_read_overflow_flags(void)
 497 {
 498         u32 val;
 499         asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val));
 500         return val;
 501 }
 502
 503 static inline void
 504 xscale2pmu_write_overflow_flags(u32 val)
 505 {
 506         asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val));
 507 }
 508
 509 static inline u32
 510 xscale2pmu_read_event_select(void)
 511 {
 512         u32 val;
 513         asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val));
 514         return val;
 515 }
 516
 517 static inline void
 518 xscale2pmu_write_event_select(u32 val)
 519 {
 520         asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val));
 521 }
 522
 523 static inline u32
 524 xscale2pmu_read_int_enable(void)
 525 {
 526         u32 val;
 527         asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val));
 528         return val;
 529 }
 530
 531 static void
 532 xscale2pmu_write_int_enable(u32 val)
 533 {
 534         asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val));
 535 }
 536
 537 static inline int
 538 xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
 539                                         enum xscale_counters counter)
 540 {
 541         int ret = 0;
 542
 543         switch (counter) {
 544         case XSCALE_CYCLE_COUNTER:
 545                 ret = of_flags & XSCALE2_CCOUNT_OVERFLOW;
 546                 break;
 547         case XSCALE_COUNTER0:
 548                 ret = of_flags & XSCALE2_COUNT0_OVERFLOW;
 549                 break;
 550         case XSCALE_COUNTER1:
 551                 ret = of_flags & XSCALE2_COUNT1_OVERFLOW;
 552                 break;
 553         case XSCALE_COUNTER2:
 554                 ret = of_flags & XSCALE2_COUNT2_OVERFLOW;
 555                 break;
 556         case XSCALE_COUNTER3:
 557                 ret = of_flags & XSCALE2_COUNT3_OVERFLOW;
 558                 break;
 559         default:
 560                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
 561         }
 562
 563         return ret;
 564 }
 565
 566 static irqreturn_t
 567 xscale2pmu_handle_irq(int irq_num, void *dev)
 568 {
 569         unsigned long pmnc, of_flags;
 570         struct perf_sample_data data;
 571         struct pmu_hw_events *cpuc;
 572         struct pt_regs *regs;
 573         int idx;
 574
 575         /* Disable the PMU. */
 576         pmnc = xscale2pmu_read_pmnc();
 577         xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
 578
 579         /* Check the overflow flag register. */
 580         of_flags = xscale2pmu_read_overflow_flags();
 581         if (!(of_flags & XSCALE2_OVERFLOWED_MASK))
 582                 return IRQ_NONE;
 583
 584         /* Clear the overflow bits. */
 585         xscale2pmu_write_overflow_flags(of_flags);
 586
 587         regs = get_irq_regs();
 588
 589         perf_sample_data_init(&data, 0);
 590
 591         cpuc = &__get_cpu_var(cpu_hw_events);
 592         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 593                 struct perf_event *event = cpuc->events[idx];
 594                 struct hw_perf_event *hwc;
 595
 596                 if (!event)
 597                         continue;
 598
 599                 if (!xscale2_pmnc_counter_has_overflowed(of_flags, idx))
 600                         continue;
 601
 602                 hwc = &event->hw;
 603                 armpmu_event_update(event, hwc, idx);
 604                 data.period = event->hw.last_period;
 605                 if (!armpmu_event_set_period(event, hwc, idx))
 606                         continue;
 607
 608                 if (perf_event_overflow(event, &data, regs))
 609                         cpu_pmu->disable(hwc, idx);
 610         }
 611
 612         irq_work_run();
 613
 614         /*
 615          * Re-enable the PMU.
 616          */
 617         pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE;
 618         xscale2pmu_write_pmnc(pmnc);
 619
 620         return IRQ_HANDLED;
 621 }
 622
 623 static void
 624 xscale2pmu_enable_event(struct hw_perf_event *hwc, int idx)
 625 {
 626         unsigned long flags, ien, evtsel;
 627         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 628
 629         ien = xscale2pmu_read_int_enable();
 630         evtsel = xscale2pmu_read_event_select();
 631
 632         switch (idx) {
 633         case XSCALE_CYCLE_COUNTER:
 634                 ien |= XSCALE2_CCOUNT_INT_EN;
 635                 break;
 636         case XSCALE_COUNTER0:
 637                 ien |= XSCALE2_COUNT0_INT_EN;
 638                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
 639                 evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT;
 640                 break;
 641         case XSCALE_COUNTER1:
 642                 ien |= XSCALE2_COUNT1_INT_EN;
 643                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
 644                 evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT;
 645                 break;
 646         case XSCALE_COUNTER2:
 647                 ien |= XSCALE2_COUNT2_INT_EN;
 648                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
 649                 evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT;
 650                 break;
 651         case XSCALE_COUNTER3:
 652                 ien |= XSCALE2_COUNT3_INT_EN;
 653                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
 654                 evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT;
 655                 break;
 656         default:
 657                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 658                 return;
 659         }
 660
 661         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 662         xscale2pmu_write_event_select(evtsel);
 663         xscale2pmu_write_int_enable(ien);
 664         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 665 }
 666
 667 static void
 668 xscale2pmu_disable_event(struct hw_perf_event *hwc, int idx)
 669 {
 670         unsigned long flags, ien, evtsel, of_flags;
 671         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 672
 673         ien = xscale2pmu_read_int_enable();
 674         evtsel = xscale2pmu_read_event_select();
 675
 676         switch (idx) {
 677         case XSCALE_CYCLE_COUNTER:
 678                 ien &= ~XSCALE2_CCOUNT_INT_EN;
 679                 of_flags = XSCALE2_CCOUNT_OVERFLOW;
 680                 break;
 681         case XSCALE_COUNTER0:
 682                 ien &= ~XSCALE2_COUNT0_INT_EN;
 683                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
 684                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT;
 685                 of_flags = XSCALE2_COUNT0_OVERFLOW;
 686                 break;
 687         case XSCALE_COUNTER1:
 688                 ien &= ~XSCALE2_COUNT1_INT_EN;
 689                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
 690                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT;
 691                 of_flags = XSCALE2_COUNT1_OVERFLOW;
 692                 break;
 693         case XSCALE_COUNTER2:
 694                 ien &= ~XSCALE2_COUNT2_INT_EN;
 695                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
 696                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT;
 697                 of_flags = XSCALE2_COUNT2_OVERFLOW;
 698                 break;
 699         case XSCALE_COUNTER3:
 700                 ien &= ~XSCALE2_COUNT3_INT_EN;
 701                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
 702                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT;
 703                 of_flags = XSCALE2_COUNT3_OVERFLOW;
 704                 break;
 705         default:
 706                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 707                 return;
 708         }
 709
 710         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 711         xscale2pmu_write_event_select(evtsel);
 712         xscale2pmu_write_int_enable(ien);
 713         xscale2pmu_write_overflow_flags(of_flags);
 714         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 715 }
 716
 717 static int
 718 xscale2pmu_get_event_idx(struct pmu_hw_events *cpuc,
 719                         struct hw_perf_event *event)
 720 {
 721         int idx = xscale1pmu_get_event_idx(cpuc, event);
 722         if (idx >= 0)
 723                 goto out;
 724
 725         if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask))
 726                 idx = XSCALE_COUNTER3;
 727         else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask))
 728                 idx = XSCALE_COUNTER2;
 729 out:
 730         return idx;
 731 }
 732
 733 static void
 734 xscale2pmu_start(void)
 735 {
 736         unsigned long flags, val;
 737         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 738
 739         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 740         val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
 741         val |= XSCALE_PMU_ENABLE;
 742         xscale2pmu_write_pmnc(val);
 743         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 744 }
 745
 746 static void
 747 xscale2pmu_stop(void)
 748 {
 749         unsigned long flags, val;
 750         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 751
 752         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 753         val = xscale2pmu_read_pmnc();
 754         val &= ~XSCALE_PMU_ENABLE;
 755         xscale2pmu_write_pmnc(val);
 756         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 757 }
 758
 759 static inline u32
 760 xscale2pmu_read_counter(int counter)
 761 {
 762         u32 val = 0;
 763
 764         switch (counter) {
 765         case XSCALE_CYCLE_COUNTER:
 766                 asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val));
 767                 break;
 768         case XSCALE_COUNTER0:
 769                 asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val));
 770                 break;
 771         case XSCALE_COUNTER1:
 772                 asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val));
 773                 break;
 774         case XSCALE_COUNTER2:
 775                 asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val));
 776                 break;
 777         case XSCALE_COUNTER3:
 778                 asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val));
 779                 break;
 780         }
 781
 782         return val;
 783 }
 784
 785 static inline void
 786 xscale2pmu_write_counter(int counter, u32 val)
 787 {
 788         switch (counter) {
 789         case XSCALE_CYCLE_COUNTER:
 790                 asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val));
 791                 break;
 792         case XSCALE_COUNTER0:
 793                 asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val));
 794                 break;
 795         case XSCALE_COUNTER1:
 796                 asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val));
 797                 break;
 798         case XSCALE_COUNTER2:
 799                 asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val));
 800                 break;
 801         case XSCALE_COUNTER3:
 802                 asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val));
 803                 break;
 804         }
 805 }
 806
 807 static struct arm_pmu xscale2pmu = {
 808         .id             = ARM_PERF_PMU_ID_XSCALE2,
 809         .name           = "xscale2",
 810         .handle_irq     = xscale2pmu_handle_irq,
 811         .enable         = xscale2pmu_enable_event,
 812         .disable        = xscale2pmu_disable_event,
 813         .read_counter   = xscale2pmu_read_counter,
 814         .write_counter  = xscale2pmu_write_counter,
 815         .get_event_idx  = xscale2pmu_get_event_idx,
 816         .start          = xscale2pmu_start,
 817         .stop           = xscale2pmu_stop,
 818         .map_event      = xscale_map_event,
 819         .num_events     = 5,
 820         .max_period     = (1LLU << 32) - 1,
 821 };
 822
 823 static struct arm_pmu *__init xscale2pmu_init(void)
 824 {
 825         return &xscale2pmu;
 826 }
 827 #else
 828 static struct arm_pmu *__init xscale1pmu_init(void)
 829 {
 830         return NULL;
 831 }
 832
 833 static struct arm_pmu *__init xscale2pmu_init(void)
 834 {
 835         return NULL;
 836 }
 837 #endif  /* CONFIG_CPU_XSCALE */