arch/arm/kernel/perf_event_v7.c

   1 /*
   2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3  *
   4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5  * 2010 (c) MontaVista Software, LLC.
   6  *
   7  * Copied from ARMv6 code, with the low level code inspired
   8  *  by the ARMv7 Oprofile code.
   9  *
  10  * Cortex-A8 has up to 4 configurable performance counters and
  11  *  a single cycle counter.
  12  * Cortex-A9 has up to 31 configurable performance counters and
  13  *  a single cycle counter.
  14  *
  15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16  *  counter and all 4 performance counters together can be reset separately.
  17  */
  18
  19 #ifdef CONFIG_CPU_V7
  20 /* Common ARMv7 event types */
  21 enum armv7_perf_types {
  22         ARMV7_PERFCTR_PMNC_SW_INCR              = 0x00,
  23         ARMV7_PERFCTR_IFETCH_MISS               = 0x01,
  24         ARMV7_PERFCTR_ITLB_MISS                 = 0x02,
  25         ARMV7_PERFCTR_DCACHE_REFILL             = 0x03,
  26         ARMV7_PERFCTR_DCACHE_ACCESS             = 0x04,
  27         ARMV7_PERFCTR_DTLB_REFILL               = 0x05,
  28         ARMV7_PERFCTR_DREAD                     = 0x06,
  29         ARMV7_PERFCTR_DWRITE                    = 0x07,
  30
  31         ARMV7_PERFCTR_EXC_TAKEN                 = 0x09,
  32         ARMV7_PERFCTR_EXC_EXECUTED              = 0x0A,
  33         ARMV7_PERFCTR_CID_WRITE                 = 0x0B,
  34         /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  35          * It counts:
  36          *  - all branch instructions,
  37          *  - instructions that explicitly write the PC,
  38          *  - exception generating instructions.
  39          */
  40         ARMV7_PERFCTR_PC_WRITE                  = 0x0C,
  41         ARMV7_PERFCTR_PC_IMM_BRANCH             = 0x0D,
  42         ARMV7_PERFCTR_UNALIGNED_ACCESS          = 0x0F,
  43         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED        = 0x10,
  44         ARMV7_PERFCTR_CLOCK_CYCLES              = 0x11,
  45
  46         ARMV7_PERFCTR_PC_BRANCH_MIS_USED        = 0x12,
  47
  48         ARMV7_PERFCTR_CPU_CYCLES                = 0xFF
  49 };
  50
  51 /* ARMv7 Cortex-A8 specific event types */
  52 enum armv7_a8_perf_types {
  53         ARMV7_PERFCTR_INSTR_EXECUTED            = 0x08,
  54
  55         ARMV7_PERFCTR_PC_PROC_RETURN            = 0x0E,
  56
  57         ARMV7_PERFCTR_WRITE_BUFFER_FULL         = 0x40,
  58         ARMV7_PERFCTR_L2_STORE_MERGED           = 0x41,
  59         ARMV7_PERFCTR_L2_STORE_BUFF             = 0x42,
  60         ARMV7_PERFCTR_L2_ACCESS                 = 0x43,
  61         ARMV7_PERFCTR_L2_CACH_MISS              = 0x44,
  62         ARMV7_PERFCTR_AXI_READ_CYCLES           = 0x45,
  63         ARMV7_PERFCTR_AXI_WRITE_CYCLES          = 0x46,
  64         ARMV7_PERFCTR_MEMORY_REPLAY             = 0x47,
  65         ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY   = 0x48,
  66         ARMV7_PERFCTR_L1_DATA_MISS              = 0x49,
  67         ARMV7_PERFCTR_L1_INST_MISS              = 0x4A,
  68         ARMV7_PERFCTR_L1_DATA_COLORING          = 0x4B,
  69         ARMV7_PERFCTR_L1_NEON_DATA              = 0x4C,
  70         ARMV7_PERFCTR_L1_NEON_CACH_DATA         = 0x4D,
  71         ARMV7_PERFCTR_L2_NEON                   = 0x4E,
  72         ARMV7_PERFCTR_L2_NEON_HIT               = 0x4F,
  73         ARMV7_PERFCTR_L1_INST                   = 0x50,
  74         ARMV7_PERFCTR_PC_RETURN_MIS_PRED        = 0x51,
  75         ARMV7_PERFCTR_PC_BRANCH_FAILED          = 0x52,
  76         ARMV7_PERFCTR_PC_BRANCH_TAKEN           = 0x53,
  77         ARMV7_PERFCTR_PC_BRANCH_EXECUTED        = 0x54,
  78         ARMV7_PERFCTR_OP_EXECUTED               = 0x55,
  79         ARMV7_PERFCTR_CYCLES_INST_STALL         = 0x56,
  80         ARMV7_PERFCTR_CYCLES_INST               = 0x57,
  81         ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL    = 0x58,
  82         ARMV7_PERFCTR_CYCLES_NEON_INST_STALL    = 0x59,
  83         ARMV7_PERFCTR_NEON_CYCLES               = 0x5A,
  84
  85         ARMV7_PERFCTR_PMU0_EVENTS               = 0x70,
  86         ARMV7_PERFCTR_PMU1_EVENTS               = 0x71,
  87         ARMV7_PERFCTR_PMU_EVENTS                = 0x72,
  88 };
  89
  90 /* ARMv7 Cortex-A9 specific event types */
  91 enum armv7_a9_perf_types {
  92         ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC     = 0x40,
  93         ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC     = 0x41,
  94         ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC       = 0x42,
  95
  96         ARMV7_PERFCTR_COHERENT_LINE_MISS        = 0x50,
  97         ARMV7_PERFCTR_COHERENT_LINE_HIT         = 0x51,
  98
  99         ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES   = 0x60,
 100         ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES   = 0x61,
 101         ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62,
 102         ARMV7_PERFCTR_STREX_EXECUTED_PASSED     = 0x63,
 103         ARMV7_PERFCTR_STREX_EXECUTED_FAILED     = 0x64,
 104         ARMV7_PERFCTR_DATA_EVICTION             = 0x65,
 105         ARMV7_PERFCTR_ISSUE_STAGE_NO_INST       = 0x66,
 106         ARMV7_PERFCTR_ISSUE_STAGE_EMPTY         = 0x67,
 107         ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE  = 0x68,
 108
 109         ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E,
 110
 111         ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST   = 0x70,
 112         ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71,
 113         ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST  = 0x72,
 114         ARMV7_PERFCTR_FP_EXECUTED_INST          = 0x73,
 115         ARMV7_PERFCTR_NEON_EXECUTED_INST        = 0x74,
 116
 117         ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80,
 118         ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES  = 0x81,
 119         ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES        = 0x82,
 120         ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES        = 0x83,
 121         ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES  = 0x84,
 122         ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES  = 0x85,
 123         ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES      = 0x86,
 124
 125         ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES  = 0x8A,
 126         ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B,
 127
 128         ARMV7_PERFCTR_ISB_INST                  = 0x90,
 129         ARMV7_PERFCTR_DSB_INST                  = 0x91,
 130         ARMV7_PERFCTR_DMB_INST                  = 0x92,
 131         ARMV7_PERFCTR_EXT_INTERRUPTS            = 0x93,
 132
 133         ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED     = 0xA0,
 134         ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED       = 0xA1,
 135         ARMV7_PERFCTR_PLE_FIFO_FLUSH            = 0xA2,
 136         ARMV7_PERFCTR_PLE_RQST_COMPLETED        = 0xA3,
 137         ARMV7_PERFCTR_PLE_FIFO_OVERFLOW         = 0xA4,
 138         ARMV7_PERFCTR_PLE_RQST_PROG             = 0xA5
 139 };
 140
 141 /*
 142  * Cortex-A8 HW events mapping
 143  *
 144  * The hardware events that we support. We do support cache operations but
 145  * we have harvard caches and no way to combine instruction and data
 146  * accesses/misses in hardware.
 147  */
 148 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 149         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 150         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 151         [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
 152         [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
 153         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 154         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 155         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 156 };
 157
 158 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 159                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 160                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 161         [C(L1D)] = {
 162                 /*
 163                  * The performance counters don't differentiate between read
 164                  * and write accesses/misses so this isn't strictly correct,
 165                  * but it's the best we can do. Writes and reads get
 166                  * combined.
 167                  */
 168                 [C(OP_READ)] = {
 169                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
 170                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
 171                 },
 172                 [C(OP_WRITE)] = {
 173                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
 174                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
 175                 },
 176                 [C(OP_PREFETCH)] = {
 177                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 178                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 179                 },
 180         },
 181         [C(L1I)] = {
 182                 [C(OP_READ)] = {
 183                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
 184                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
 185                 },
 186                 [C(OP_WRITE)] = {
 187                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
 188                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
 189                 },
 190                 [C(OP_PREFETCH)] = {
 191                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 192                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 193                 },
 194         },
 195         [C(LL)] = {
 196                 [C(OP_READ)] = {
 197                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
 198                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
 199                 },
 200                 [C(OP_WRITE)] = {
 201                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
 202                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
 203                 },
 204                 [C(OP_PREFETCH)] = {
 205                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 206                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 207                 },
 208         },
 209         [C(DTLB)] = {
 210                 /*
 211                  * Only ITLB misses and DTLB refills are supported.
 212                  * If users want the DTLB refills misses a raw counter
 213                  * must be used.
 214                  */
 215                 [C(OP_READ)] = {
 216                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 217                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 218                 },
 219                 [C(OP_WRITE)] = {
 220                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 221                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 222                 },
 223                 [C(OP_PREFETCH)] = {
 224                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 225                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 226                 },
 227         },
 228         [C(ITLB)] = {
 229                 [C(OP_READ)] = {
 230                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 231                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
 232                 },
 233                 [C(OP_WRITE)] = {
 234                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 235                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
 236                 },
 237                 [C(OP_PREFETCH)] = {
 238                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 239                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 240                 },
 241         },
 242         [C(BPU)] = {
 243                 [C(OP_READ)] = {
 244                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
 245                         [C(RESULT_MISS)]
 246                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 247                 },
 248                 [C(OP_WRITE)] = {
 249                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
 250                         [C(RESULT_MISS)]
 251                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 252                 },
 253                 [C(OP_PREFETCH)] = {
 254                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 255                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 256                 },
 257         },
 258         [C(NODE)] = {
 259                 [C(OP_READ)] = {
 260                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 261                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 262                 },
 263                 [C(OP_WRITE)] = {
 264                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 265                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 266                 },
 267                 [C(OP_PREFETCH)] = {
 268                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 269                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 270                 },
 271         },
 272 };
 273
 274 /*
 275  * Cortex-A9 HW events mapping
 276  */
 277 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 278         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 279         [PERF_COUNT_HW_INSTRUCTIONS]        =
 280                                         ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE,
 281         [PERF_COUNT_HW_CACHE_REFERENCES]    = ARMV7_PERFCTR_COHERENT_LINE_HIT,
 282         [PERF_COUNT_HW_CACHE_MISSES]        = ARMV7_PERFCTR_COHERENT_LINE_MISS,
 283         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 284         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 285         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 286 };
 287
 288 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 289                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 290                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 291         [C(L1D)] = {
 292                 /*
 293                  * The performance counters don't differentiate between read
 294                  * and write accesses/misses so this isn't strictly correct,
 295                  * but it's the best we can do. Writes and reads get
 296                  * combined.
 297                  */
 298                 [C(OP_READ)] = {
 299                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
 300                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
 301                 },
 302                 [C(OP_WRITE)] = {
 303                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
 304                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
 305                 },
 306                 [C(OP_PREFETCH)] = {
 307                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 308                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 309                 },
 310         },
 311         [C(L1I)] = {
 312                 [C(OP_READ)] = {
 313                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 314                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
 315                 },
 316                 [C(OP_WRITE)] = {
 317                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 318                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
 319                 },
 320                 [C(OP_PREFETCH)] = {
 321                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 322                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 323                 },
 324         },
 325         [C(LL)] = {
 326                 [C(OP_READ)] = {
 327                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 328                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 329                 },
 330                 [C(OP_WRITE)] = {
 331                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 332                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 333                 },
 334                 [C(OP_PREFETCH)] = {
 335                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 336                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 337                 },
 338         },
 339         [C(DTLB)] = {
 340                 /*
 341                  * Only ITLB misses and DTLB refills are supported.
 342                  * If users want the DTLB refills misses a raw counter
 343                  * must be used.
 344                  */
 345                 [C(OP_READ)] = {
 346                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 347                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 348                 },
 349                 [C(OP_WRITE)] = {
 350                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 351                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 352                 },
 353                 [C(OP_PREFETCH)] = {
 354                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 355                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 356                 },
 357         },
 358         [C(ITLB)] = {
 359                 [C(OP_READ)] = {
 360                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 361                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
 362                 },
 363                 [C(OP_WRITE)] = {
 364                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 365                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
 366                 },
 367                 [C(OP_PREFETCH)] = {
 368                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 369                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 370                 },
 371         },
 372         [C(BPU)] = {
 373                 [C(OP_READ)] = {
 374                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
 375                         [C(RESULT_MISS)]
 376                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 377                 },
 378                 [C(OP_WRITE)] = {
 379                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
 380                         [C(RESULT_MISS)]
 381                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 382                 },
 383                 [C(OP_PREFETCH)] = {
 384                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 385                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 386                 },
 387         },
 388         [C(NODE)] = {
 389                 [C(OP_READ)] = {
 390                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 391                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 392                 },
 393                 [C(OP_WRITE)] = {
 394                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 395                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 396                 },
 397                 [C(OP_PREFETCH)] = {
 398                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 399                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 400                 },
 401         },
 402 };
 403
 404 /*
 405  * Perf Events counters
 406  */
 407 enum armv7_counters {
 408         ARMV7_CYCLE_COUNTER             = 1,    /* Cycle counter */
 409         ARMV7_COUNTER0                  = 2,    /* First event counter */
 410 };
 411
 412 /*
 413  * The cycle counter is ARMV7_CYCLE_COUNTER.
 414  * The first event counter is ARMV7_COUNTER0.
 415  * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1).
 416  */
 417 #define ARMV7_COUNTER_LAST      (ARMV7_COUNTER0 + armpmu->num_events - 1)
 418
 419 /*
 420  * ARMv7 low level PMNC access
 421  */
 422
 423 /*
 424  * Per-CPU PMNC: config reg
 425  */
 426 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
 427 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
 428 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
 429 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 430 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
 431 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 432 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
 433 #define ARMV7_PMNC_N_MASK       0x1f
 434 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
 435
 436 /*
 437  * Available counters
 438  */
 439 #define ARMV7_CNT0              0       /* First event counter */
 440 #define ARMV7_CCNT              31      /* Cycle counter */
 441
 442 /* Perf Event to low level counters mapping */
 443 #define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0)
 444
 445 /*
 446  * CNTENS: counters enable reg
 447  */
 448 #define ARMV7_CNTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 449 #define ARMV7_CNTENS_C          (1 << ARMV7_CCNT)
 450
 451 /*
 452  * CNTENC: counters disable reg
 453  */
 454 #define ARMV7_CNTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 455 #define ARMV7_CNTENC_C          (1 << ARMV7_CCNT)
 456
 457 /*
 458  * INTENS: counters overflow interrupt enable reg
 459  */
 460 #define ARMV7_INTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 461 #define ARMV7_INTENS_C          (1 << ARMV7_CCNT)
 462
 463 /*
 464  * INTENC: counters overflow interrupt disable reg
 465  */
 466 #define ARMV7_INTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 467 #define ARMV7_INTENC_C          (1 << ARMV7_CCNT)
 468
 469 /*
 470  * EVTSEL: Event selection reg
 471  */
 472 #define ARMV7_EVTSEL_MASK       0xff            /* Mask for writable bits */
 473
 474 /*
 475  * SELECT: Counter selection reg
 476  */
 477 #define ARMV7_SELECT_MASK       0x1f            /* Mask for writable bits */
 478
 479 /*
 480  * FLAG: counters overflow flag status reg
 481  */
 482 #define ARMV7_FLAG_P(idx)       (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 483 #define ARMV7_FLAG_C            (1 << ARMV7_CCNT)
 484 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
 485 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
 486
 487 static inline unsigned long armv7_pmnc_read(void)
 488 {
 489         u32 val;
 490         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 491         return val;
 492 }
 493
 494 static inline void armv7_pmnc_write(unsigned long val)
 495 {
 496         val &= ARMV7_PMNC_MASK;
 497         isb();
 498         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 499 }
 500
 501 static inline int armv7_pmnc_has_overflowed(unsigned long pmnc)
 502 {
 503         return pmnc & ARMV7_OVERFLOWED_MASK;
 504 }
 505
 506 static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc,
 507                                         enum armv7_counters counter)
 508 {
 509         int ret = 0;
 510
 511         if (counter == ARMV7_CYCLE_COUNTER)
 512                 ret = pmnc & ARMV7_FLAG_C;
 513         else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST))
 514                 ret = pmnc & ARMV7_FLAG_P(counter);
 515         else
 516                 pr_err("CPU%u checking wrong counter %d overflow status\n",
 517                         smp_processor_id(), counter);
 518
 519         return ret;
 520 }
 521
 522 static inline int armv7_pmnc_select_counter(unsigned int idx)
 523 {
 524         u32 val;
 525
 526         if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) {
 527                 pr_err("CPU%u selecting wrong PMNC counter"
 528                         " %d\n", smp_processor_id(), idx);
 529                 return -1;
 530         }
 531
 532         val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK;
 533         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val));
 534         isb();
 535
 536         return idx;
 537 }
 538
 539 static inline u32 armv7pmu_read_counter(int idx)
 540 {
 541         unsigned long value = 0;
 542
 543         if (idx == ARMV7_CYCLE_COUNTER)
 544                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 545         else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
 546                 if (armv7_pmnc_select_counter(idx) == idx)
 547                         asm volatile("mrc p15, 0, %0, c9, c13, 2"
 548                                      : "=r" (value));
 549         } else
 550                 pr_err("CPU%u reading wrong counter %d\n",
 551                         smp_processor_id(), idx);
 552
 553         return value;
 554 }
 555
 556 static inline void armv7pmu_write_counter(int idx, u32 value)
 557 {
 558         if (idx == ARMV7_CYCLE_COUNTER)
 559                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 560         else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
 561                 if (armv7_pmnc_select_counter(idx) == idx)
 562                         asm volatile("mcr p15, 0, %0, c9, c13, 2"
 563                                      : : "r" (value));
 564         } else
 565                 pr_err("CPU%u writing wrong counter %d\n",
 566                         smp_processor_id(), idx);
 567 }
 568
 569 static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val)
 570 {
 571         if (armv7_pmnc_select_counter(idx) == idx) {
 572                 val &= ARMV7_EVTSEL_MASK;
 573                 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 574         }
 575 }
 576
 577 static inline u32 armv7_pmnc_enable_counter(unsigned int idx)
 578 {
 579         u32 val;
 580
 581         if ((idx != ARMV7_CYCLE_COUNTER) &&
 582             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
 583                 pr_err("CPU%u enabling wrong PMNC counter"
 584                         " %d\n", smp_processor_id(), idx);
 585                 return -1;
 586         }
 587
 588         if (idx == ARMV7_CYCLE_COUNTER)
 589                 val = ARMV7_CNTENS_C;
 590         else
 591                 val = ARMV7_CNTENS_P(idx);
 592
 593         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val));
 594
 595         return idx;
 596 }
 597
 598 static inline u32 armv7_pmnc_disable_counter(unsigned int idx)
 599 {
 600         u32 val;
 601
 602
 603         if ((idx != ARMV7_CYCLE_COUNTER) &&
 604             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
 605                 pr_err("CPU%u disabling wrong PMNC counter"
 606                         " %d\n", smp_processor_id(), idx);
 607                 return -1;
 608         }
 609
 610         if (idx == ARMV7_CYCLE_COUNTER)
 611                 val = ARMV7_CNTENC_C;
 612         else
 613                 val = ARMV7_CNTENC_P(idx);
 614
 615         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val));
 616
 617         return idx;
 618 }
 619
 620 static inline u32 armv7_pmnc_enable_intens(unsigned int idx)
 621 {
 622         u32 val;
 623
 624         if ((idx != ARMV7_CYCLE_COUNTER) &&
 625             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
 626                 pr_err("CPU%u enabling wrong PMNC counter"
 627                         " interrupt enable %d\n", smp_processor_id(), idx);
 628                 return -1;
 629         }
 630
 631         if (idx == ARMV7_CYCLE_COUNTER)
 632                 val = ARMV7_INTENS_C;
 633         else
 634                 val = ARMV7_INTENS_P(idx);
 635
 636         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
 637
 638         return idx;
 639 }
 640
 641 static inline u32 armv7_pmnc_disable_intens(unsigned int idx)
 642 {
 643         u32 val;
 644
 645         if ((idx != ARMV7_CYCLE_COUNTER) &&
 646             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
 647                 pr_err("CPU%u disabling wrong PMNC counter"
 648                         " interrupt enable %d\n", smp_processor_id(), idx);
 649                 return -1;
 650         }
 651
 652         if (idx == ARMV7_CYCLE_COUNTER)
 653                 val = ARMV7_INTENC_C;
 654         else
 655                 val = ARMV7_INTENC_P(idx);
 656
 657         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
 658
 659         return idx;
 660 }
 661
 662 static inline u32 armv7_pmnc_getreset_flags(void)
 663 {
 664         u32 val;
 665
 666         /* Read */
 667         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 668
 669         /* Write to clear flags */
 670         val &= ARMV7_FLAG_MASK;
 671         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 672
 673         return val;
 674 }
 675
 676 #ifdef DEBUG
 677 static void armv7_pmnc_dump_regs(void)
 678 {
 679         u32 val;
 680         unsigned int cnt;
 681
 682         printk(KERN_INFO "PMNC registers dump:\n");
 683
 684         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 685         printk(KERN_INFO "PMNC  =0x%08x\n", val);
 686
 687         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 688         printk(KERN_INFO "CNTENS=0x%08x\n", val);
 689
 690         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 691         printk(KERN_INFO "INTENS=0x%08x\n", val);
 692
 693         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 694         printk(KERN_INFO "FLAGS =0x%08x\n", val);
 695
 696         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 697         printk(KERN_INFO "SELECT=0x%08x\n", val);
 698
 699         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 700         printk(KERN_INFO "CCNT  =0x%08x\n", val);
 701
 702         for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) {
 703                 armv7_pmnc_select_counter(cnt);
 704                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 705                 printk(KERN_INFO "CNT[%d] count =0x%08x\n",
 706                         cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
 707                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 708                 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
 709                         cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
 710         }
 711 }
 712 #endif
 713
 714 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
 715 {
 716         unsigned long flags;
 717
 718         /*
 719          * Enable counter and interrupt, and set the counter to count
 720          * the event that we're interested in.
 721          */
 722         raw_spin_lock_irqsave(&pmu_lock, flags);
 723
 724         /*
 725          * Disable counter
 726          */
 727         armv7_pmnc_disable_counter(idx);
 728
 729         /*
 730          * Set event (if destined for PMNx counters)
 731          * We don't need to set the event if it's a cycle count
 732          */
 733         if (idx != ARMV7_CYCLE_COUNTER)
 734                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
 735
 736         /*
 737          * Enable interrupt for this counter
 738          */
 739         armv7_pmnc_enable_intens(idx);
 740
 741         /*
 742          * Enable counter
 743          */
 744         armv7_pmnc_enable_counter(idx);
 745
 746         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 747 }
 748
 749 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
 750 {
 751         unsigned long flags;
 752
 753         /*
 754          * Disable counter and interrupt
 755          */
 756         raw_spin_lock_irqsave(&pmu_lock, flags);
 757
 758         /*
 759          * Disable counter
 760          */
 761         armv7_pmnc_disable_counter(idx);
 762
 763         /*
 764          * Disable interrupt for this counter
 765          */
 766         armv7_pmnc_disable_intens(idx);
 767
 768         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 769 }
 770
 771 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
 772 {
 773         unsigned long pmnc;
 774         struct perf_sample_data data;
 775         struct cpu_hw_events *cpuc;
 776         struct pt_regs *regs;
 777         int idx;
 778
 779         /*
 780          * Get and reset the IRQ flags
 781          */
 782         pmnc = armv7_pmnc_getreset_flags();
 783
 784         /*
 785          * Did an overflow occur?
 786          */
 787         if (!armv7_pmnc_has_overflowed(pmnc))
 788                 return IRQ_NONE;
 789
 790         /*
 791          * Handle the counter(s) overflow(s)
 792          */
 793         regs = get_irq_regs();
 794
 795         perf_sample_data_init(&data, 0);
 796
 797         cpuc = &__get_cpu_var(cpu_hw_events);
 798         for (idx = 0; idx <= armpmu->num_events; ++idx) {
 799                 struct perf_event *event = cpuc->events[idx];
 800                 struct hw_perf_event *hwc;
 801
 802                 if (!test_bit(idx, cpuc->active_mask))
 803                         continue;
 804
 805                 /*
 806                  * We have a single interrupt for all counters. Check that
 807                  * each counter has overflowed before we process it.
 808                  */
 809                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
 810                         continue;
 811
 812                 hwc = &event->hw;
 813                 armpmu_event_update(event, hwc, idx, 1);
 814                 data.period = event->hw.last_period;
 815                 if (!armpmu_event_set_period(event, hwc, idx))
 816                         continue;
 817
 818                 if (perf_event_overflow(event, &data, regs))
 819                         armpmu->disable(hwc, idx);
 820         }
 821
 822         /*
 823          * Handle the pending perf events.
 824          *
 825          * Note: this call *must* be run with interrupts disabled. For
 826          * platforms that can have the PMU interrupts raised as an NMI, this
 827          * will not work.
 828          */
 829         irq_work_run();
 830
 831         return IRQ_HANDLED;
 832 }
 833
 834 static void armv7pmu_start(void)
 835 {
 836         unsigned long flags;
 837
 838         raw_spin_lock_irqsave(&pmu_lock, flags);
 839         /* Enable all counters */
 840         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
 841         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 842 }
 843
 844 static void armv7pmu_stop(void)
 845 {
 846         unsigned long flags;
 847
 848         raw_spin_lock_irqsave(&pmu_lock, flags);
 849         /* Disable all counters */
 850         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
 851         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 852 }
 853
 854 static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc,
 855                                   struct hw_perf_event *event)
 856 {
 857         int idx;
 858
 859         /* Always place a cycle counter into the cycle counter. */
 860         if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) {
 861                 if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask))
 862                         return -EAGAIN;
 863
 864                 return ARMV7_CYCLE_COUNTER;
 865         } else {
 866                 /*
 867                  * For anything other than a cycle counter, try and use
 868                  * the events counters
 869                  */
 870                 for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) {
 871                         if (!test_and_set_bit(idx, cpuc->used_mask))
 872                                 return idx;
 873                 }
 874
 875                 /* The counters are all in use. */
 876                 return -EAGAIN;
 877         }
 878 }
 879
 880 static void armv7pmu_reset(void *info)
 881 {
 882         u32 idx, nb_cnt = armpmu->num_events;
 883
 884         /* The counter and interrupt enable registers are unknown at reset. */
 885         for (idx = 1; idx < nb_cnt; ++idx)
 886                 armv7pmu_disable_event(NULL, idx);
 887
 888         /* Initialize & Reset PMNC: C and P bits */
 889         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
 890 }
 891
 892 static struct arm_pmu armv7pmu = {
 893         .handle_irq             = armv7pmu_handle_irq,
 894         .enable                 = armv7pmu_enable_event,
 895         .disable                = armv7pmu_disable_event,
 896         .read_counter           = armv7pmu_read_counter,
 897         .write_counter          = armv7pmu_write_counter,
 898         .get_event_idx          = armv7pmu_get_event_idx,
 899         .start                  = armv7pmu_start,
 900         .stop                   = armv7pmu_stop,
 901         .reset                  = armv7pmu_reset,
 902         .raw_event_mask         = 0xFF,
 903         .max_period             = (1LLU << 32) - 1,
 904 };
 905
 906 static u32 __init armv7_read_num_pmnc_events(void)
 907 {
 908         u32 nb_cnt;
 909
 910         /* Read the nb of CNTx counters supported from PMNC */
 911         nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
 912
 913         /* Add the CPU cycles counter and return */
 914         return nb_cnt + 1;
 915 }
 916
 917 static const struct arm_pmu *__init armv7_a8_pmu_init(void)
 918 {
 919         armv7pmu.id             = ARM_PERF_PMU_ID_CA8;
 920         armv7pmu.name           = "ARMv7 Cortex-A8";
 921         armv7pmu.cache_map      = &armv7_a8_perf_cache_map;
 922         armv7pmu.event_map      = &armv7_a8_perf_map;
 923         armv7pmu.num_events     = armv7_read_num_pmnc_events();
 924         return &armv7pmu;
 925 }
 926
 927 static const struct arm_pmu *__init armv7_a9_pmu_init(void)
 928 {
 929         armv7pmu.id             = ARM_PERF_PMU_ID_CA9;
 930         armv7pmu.name           = "ARMv7 Cortex-A9";
 931         armv7pmu.cache_map      = &armv7_a9_perf_cache_map;
 932         armv7pmu.event_map      = &armv7_a9_perf_map;
 933         armv7pmu.num_events     = armv7_read_num_pmnc_events();
 934         return &armv7pmu;
 935 }
 936 #else
 937 static const struct arm_pmu *__init armv7_a8_pmu_init(void)
 938 {
 939         return NULL;
 940 }
 941
 942 static const struct arm_pmu *__init armv7_a9_pmu_init(void)
 943 {
 944         return NULL;
 945 }
 946 #endif  /* CONFIG_CPU_V7 */