tools/perf/util/evsel.c

   1 /*
   2  * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
   3  *
   4  * Parts came from builtin-{top,stat,record}.c, see those files for further
   5  * copyright notes.
   6  *
   7  * Released under the GPL v2. (and only v2, not any later version)
   8  */
   9
  10 #include <byteswap.h>
  11 #include <linux/bitops.h>
  12 #include "asm/bug.h"
  13 #include "debugfs.h"
  14 #include "event-parse.h"
  15 #include "evsel.h"
  16 #include "evlist.h"
  17 #include "util.h"
  18 #include "cpumap.h"
  19 #include "thread_map.h"
  20 #include "target.h"
  21 #include "../../../include/linux/hw_breakpoint.h"
  22 #include "../../include/linux/perf_event.h"
  23 #include "perf_regs.h"
  24
  25 #define FD(e, x, y) (*(int *)xyarray__entry(e->fd, x, y))
  26
  27 static int __perf_evsel__sample_size(u64 sample_type)
  28 {
  29         u64 mask = sample_type & PERF_SAMPLE_MASK;
  30         int size = 0;
  31         int i;
  32
  33         for (i = 0; i < 64; i++) {
  34                 if (mask & (1ULL << i))
  35                         size++;
  36         }
  37
  38         size *= sizeof(u64);
  39
  40         return size;
  41 }
  42
  43 void hists__init(struct hists *hists)
  44 {
  45         memset(hists, 0, sizeof(*hists));
  46         hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT;
  47         hists->entries_in = &hists->entries_in_array[0];
  48         hists->entries_collapsed = RB_ROOT;
  49         hists->entries = RB_ROOT;
  50         pthread_mutex_init(&hists->lock, NULL);
  51 }
  52
  53 void perf_evsel__init(struct perf_evsel *evsel,
  54                       struct perf_event_attr *attr, int idx)
  55 {
  56         evsel->idx         = idx;
  57         evsel->attr        = *attr;
  58         INIT_LIST_HEAD(&evsel->node);
  59         hists__init(&evsel->hists);
  60         evsel->sample_size = __perf_evsel__sample_size(attr->sample_type);
  61 }
  62
  63 struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr, int idx)
  64 {
  65         struct perf_evsel *evsel = zalloc(sizeof(*evsel));
  66
  67         if (evsel != NULL)
  68                 perf_evsel__init(evsel, attr, idx);
  69
  70         return evsel;
  71 }
  72
  73 static struct event_format *event_format__new(const char *sys, const char *name)
  74 {
  75         int fd, n;
  76         char *filename;
  77         void *bf = NULL, *nbf;
  78         size_t size = 0, alloc_size = 0;
  79         struct event_format *format = NULL;
  80
  81         if (asprintf(&filename, "%s/%s/%s/format", tracing_events_path, sys, name) < 0)
  82                 goto out;
  83
  84         fd = open(filename, O_RDONLY);
  85         if (fd < 0)
  86                 goto out_free_filename;
  87
  88         do {
  89                 if (size == alloc_size) {
  90                         alloc_size += BUFSIZ;
  91                         nbf = realloc(bf, alloc_size);
  92                         if (nbf == NULL)
  93                                 goto out_free_bf;
  94                         bf = nbf;
  95                 }
  96
  97                 n = read(fd, bf + size, BUFSIZ);
  98                 if (n < 0)
  99                         goto out_free_bf;
 100                 size += n;
 101         } while (n > 0);
 102
 103         pevent_parse_format(&format, bf, size, sys);
 104
 105 out_free_bf:
 106         free(bf);
 107         close(fd);
 108 out_free_filename:
 109         free(filename);
 110 out:
 111         return format;
 112 }
 113
 114 struct perf_evsel *perf_evsel__newtp(const char *sys, const char *name, int idx)
 115 {
 116         struct perf_evsel *evsel = zalloc(sizeof(*evsel));
 117
 118         if (evsel != NULL) {
 119                 struct perf_event_attr attr = {
 120                         .type          = PERF_TYPE_TRACEPOINT,
 121                         .sample_type   = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
 122                                           PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
 123                 };
 124
 125                 evsel->tp_format = event_format__new(sys, name);
 126                 if (evsel->tp_format == NULL)
 127                         goto out_free;
 128
 129                 event_attr_init(&attr);
 130                 attr.config = evsel->tp_format->id;
 131                 attr.sample_period = 1;
 132                 perf_evsel__init(evsel, &attr, idx);
 133                 evsel->name = evsel->tp_format->name;
 134         }
 135
 136         return evsel;
 137
 138 out_free:
 139         free(evsel);
 140         return NULL;
 141 }
 142
 143 const char *perf_evsel__hw_names[PERF_COUNT_HW_MAX] = {
 144         "cycles",
 145         "instructions",
 146         "cache-references",
 147         "cache-misses",
 148         "branches",
 149         "branch-misses",
 150         "bus-cycles",
 151         "stalled-cycles-frontend",
 152         "stalled-cycles-backend",
 153         "ref-cycles",
 154 };
 155
 156 static const char *__perf_evsel__hw_name(u64 config)
 157 {
 158         if (config < PERF_COUNT_HW_MAX && perf_evsel__hw_names[config])
 159                 return perf_evsel__hw_names[config];
 160
 161         return "unknown-hardware";
 162 }
 163
 164 static int perf_evsel__add_modifiers(struct perf_evsel *evsel, char *bf, size_t size)
 165 {
 166         int colon = 0, r = 0;
 167         struct perf_event_attr *attr = &evsel->attr;
 168         bool exclude_guest_default = false;
 169
 170 #define MOD_PRINT(context, mod) do {                                    \
 171                 if (!attr->exclude_##context) {                         \
 172                         if (!colon) colon = ++r;                        \
 173                         r += scnprintf(bf + r, size - r, "%c", mod);    \
 174                 } } while(0)
 175
 176         if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
 177                 MOD_PRINT(kernel, 'k');
 178                 MOD_PRINT(user, 'u');
 179                 MOD_PRINT(hv, 'h');
 180                 exclude_guest_default = true;
 181         }
 182
 183         if (attr->precise_ip) {
 184                 if (!colon)
 185                         colon = ++r;
 186                 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
 187                 exclude_guest_default = true;
 188         }
 189
 190         if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
 191                 MOD_PRINT(host, 'H');
 192                 MOD_PRINT(guest, 'G');
 193         }
 194 #undef MOD_PRINT
 195         if (colon)
 196                 bf[colon - 1] = ':';
 197         return r;
 198 }
 199
 200 static int perf_evsel__hw_name(struct perf_evsel *evsel, char *bf, size_t size)
 201 {
 202         int r = scnprintf(bf, size, "%s", __perf_evsel__hw_name(evsel->attr.config));
 203         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 204 }
 205
 206 const char *perf_evsel__sw_names[PERF_COUNT_SW_MAX] = {
 207         "cpu-clock",
 208         "task-clock",
 209         "page-faults",
 210         "context-switches",
 211         "cpu-migrations",
 212         "minor-faults",
 213         "major-faults",
 214         "alignment-faults",
 215         "emulation-faults",
 216 };
 217
 218 static const char *__perf_evsel__sw_name(u64 config)
 219 {
 220         if (config < PERF_COUNT_SW_MAX && perf_evsel__sw_names[config])
 221                 return perf_evsel__sw_names[config];
 222         return "unknown-software";
 223 }
 224
 225 static int perf_evsel__sw_name(struct perf_evsel *evsel, char *bf, size_t size)
 226 {
 227         int r = scnprintf(bf, size, "%s", __perf_evsel__sw_name(evsel->attr.config));
 228         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 229 }
 230
 231 static int __perf_evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
 232 {
 233         int r;
 234
 235         r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
 236
 237         if (type & HW_BREAKPOINT_R)
 238                 r += scnprintf(bf + r, size - r, "r");
 239
 240         if (type & HW_BREAKPOINT_W)
 241                 r += scnprintf(bf + r, size - r, "w");
 242
 243         if (type & HW_BREAKPOINT_X)
 244                 r += scnprintf(bf + r, size - r, "x");
 245
 246         return r;
 247 }
 248
 249 static int perf_evsel__bp_name(struct perf_evsel *evsel, char *bf, size_t size)
 250 {
 251         struct perf_event_attr *attr = &evsel->attr;
 252         int r = __perf_evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
 253         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 254 }
 255
 256 const char *perf_evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX]
 257                                 [PERF_EVSEL__MAX_ALIASES] = {
 258  { "L1-dcache", "l1-d",         "l1d",          "L1-data",              },
 259  { "L1-icache", "l1-i",         "l1i",          "L1-instruction",       },
 260  { "LLC",       "L2",                                                   },
 261  { "dTLB",      "d-tlb",        "Data-TLB",                             },
 262  { "iTLB",      "i-tlb",        "Instruction-TLB",                      },
 263  { "branch",    "branches",     "bpu",          "btb",          "bpc",  },
 264  { "node",                                                              },
 265 };
 266
 267 const char *perf_evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX]
 268                                    [PERF_EVSEL__MAX_ALIASES] = {
 269  { "load",      "loads",        "read",                                 },
 270  { "store",     "stores",       "write",                                },
 271  { "prefetch",  "prefetches",   "speculative-read", "speculative-load", },
 272 };
 273
 274 const char *perf_evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX]
 275                                        [PERF_EVSEL__MAX_ALIASES] = {
 276  { "refs",      "Reference",    "ops",          "access",               },
 277  { "misses",    "miss",                                                 },
 278 };
 279
 280 #define C(x)            PERF_COUNT_HW_CACHE_##x
 281 #define CACHE_READ      (1 << C(OP_READ))
 282 #define CACHE_WRITE     (1 << C(OP_WRITE))
 283 #define CACHE_PREFETCH  (1 << C(OP_PREFETCH))
 284 #define COP(x)          (1 << x)
 285
 286 /*
 287  * cache operartion stat
 288  * L1I : Read and prefetch only
 289  * ITLB and BPU : Read-only
 290  */
 291 static unsigned long perf_evsel__hw_cache_stat[C(MAX)] = {
 292  [C(L1D)]       = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 293  [C(L1I)]       = (CACHE_READ | CACHE_PREFETCH),
 294  [C(LL)]        = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 295  [C(DTLB)]      = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 296  [C(ITLB)]      = (CACHE_READ),
 297  [C(BPU)]       = (CACHE_READ),
 298  [C(NODE)]      = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 299 };
 300
 301 bool perf_evsel__is_cache_op_valid(u8 type, u8 op)
 302 {
 303         if (perf_evsel__hw_cache_stat[type] & COP(op))
 304                 return true;    /* valid */
 305         else
 306                 return false;   /* invalid */
 307 }
 308
 309 int __perf_evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result,
 310                                             char *bf, size_t size)
 311 {
 312         if (result) {
 313                 return scnprintf(bf, size, "%s-%s-%s", perf_evsel__hw_cache[type][0],
 314                                  perf_evsel__hw_cache_op[op][0],
 315                                  perf_evsel__hw_cache_result[result][0]);
 316         }
 317
 318         return scnprintf(bf, size, "%s-%s", perf_evsel__hw_cache[type][0],
 319                          perf_evsel__hw_cache_op[op][1]);
 320 }
 321
 322 static int __perf_evsel__hw_cache_name(u64 config, char *bf, size_t size)
 323 {
 324         u8 op, result, type = (config >>  0) & 0xff;
 325         const char *err = "unknown-ext-hardware-cache-type";
 326
 327         if (type > PERF_COUNT_HW_CACHE_MAX)
 328                 goto out_err;
 329
 330         op = (config >>  8) & 0xff;
 331         err = "unknown-ext-hardware-cache-op";
 332         if (op > PERF_COUNT_HW_CACHE_OP_MAX)
 333                 goto out_err;
 334
 335         result = (config >> 16) & 0xff;
 336         err = "unknown-ext-hardware-cache-result";
 337         if (result > PERF_COUNT_HW_CACHE_RESULT_MAX)
 338                 goto out_err;
 339
 340         err = "invalid-cache";
 341         if (!perf_evsel__is_cache_op_valid(type, op))
 342                 goto out_err;
 343
 344         return __perf_evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
 345 out_err:
 346         return scnprintf(bf, size, "%s", err);
 347 }
 348
 349 static int perf_evsel__hw_cache_name(struct perf_evsel *evsel, char *bf, size_t size)
 350 {
 351         int ret = __perf_evsel__hw_cache_name(evsel->attr.config, bf, size);
 352         return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
 353 }
 354
 355 static int perf_evsel__raw_name(struct perf_evsel *evsel, char *bf, size_t size)
 356 {
 357         int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->attr.config);
 358         return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
 359 }
 360
 361 const char *perf_evsel__name(struct perf_evsel *evsel)
 362 {
 363         char bf[128];
 364
 365         if (evsel->name)
 366                 return evsel->name;
 367
 368         switch (evsel->attr.type) {
 369         case PERF_TYPE_RAW:
 370                 perf_evsel__raw_name(evsel, bf, sizeof(bf));
 371                 break;
 372
 373         case PERF_TYPE_HARDWARE:
 374                 perf_evsel__hw_name(evsel, bf, sizeof(bf));
 375                 break;
 376
 377         case PERF_TYPE_HW_CACHE:
 378                 perf_evsel__hw_cache_name(evsel, bf, sizeof(bf));
 379                 break;
 380
 381         case PERF_TYPE_SOFTWARE:
 382                 perf_evsel__sw_name(evsel, bf, sizeof(bf));
 383                 break;
 384
 385         case PERF_TYPE_TRACEPOINT:
 386                 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
 387                 break;
 388
 389         case PERF_TYPE_BREAKPOINT:
 390                 perf_evsel__bp_name(evsel, bf, sizeof(bf));
 391                 break;
 392
 393         default:
 394                 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
 395                           evsel->attr.type);
 396                 break;
 397         }
 398
 399         evsel->name = strdup(bf);
 400
 401         return evsel->name ?: "unknown";
 402 }
 403
 404 void perf_evsel__config(struct perf_evsel *evsel, struct perf_record_opts *opts,
 405                         struct perf_evsel *first)
 406 {
 407         struct perf_event_attr *attr = &evsel->attr;
 408         int track = !evsel->idx; /* only the first counter needs these */
 409
 410         attr->disabled = 1;
 411         attr->sample_id_all = opts->sample_id_all_missing ? 0 : 1;
 412         attr->inherit       = !opts->no_inherit;
 413         attr->read_format   = PERF_FORMAT_TOTAL_TIME_ENABLED |
 414                               PERF_FORMAT_TOTAL_TIME_RUNNING |
 415                               PERF_FORMAT_ID;
 416
 417         attr->sample_type  |= PERF_SAMPLE_IP | PERF_SAMPLE_TID;
 418
 419         /*
 420          * We default some events to a 1 default interval. But keep
 421          * it a weak assumption overridable by the user.
 422          */
 423         if (!attr->sample_period || (opts->user_freq != UINT_MAX &&
 424                                      opts->user_interval != ULLONG_MAX)) {
 425                 if (opts->freq) {
 426                         attr->sample_type       |= PERF_SAMPLE_PERIOD;
 427                         attr->freq              = 1;
 428                         attr->sample_freq       = opts->freq;
 429                 } else {
 430                         attr->sample_period = opts->default_interval;
 431                 }
 432         }
 433
 434         if (opts->no_samples)
 435                 attr->sample_freq = 0;
 436
 437         if (opts->inherit_stat)
 438                 attr->inherit_stat = 1;
 439
 440         if (opts->sample_address) {
 441                 attr->sample_type       |= PERF_SAMPLE_ADDR;
 442                 attr->mmap_data = track;
 443         }
 444
 445         if (opts->call_graph) {
 446                 attr->sample_type       |= PERF_SAMPLE_CALLCHAIN;
 447
 448                 if (opts->call_graph == CALLCHAIN_DWARF) {
 449                         attr->sample_type |= PERF_SAMPLE_REGS_USER |
 450                                              PERF_SAMPLE_STACK_USER;
 451                         attr->sample_regs_user = PERF_REGS_MASK;
 452                         attr->sample_stack_user = opts->stack_dump_size;
 453                         attr->exclude_callchain_user = 1;
 454                 }
 455         }
 456
 457         if (perf_target__has_cpu(&opts->target))
 458                 attr->sample_type       |= PERF_SAMPLE_CPU;
 459
 460         if (opts->period)
 461                 attr->sample_type       |= PERF_SAMPLE_PERIOD;
 462
 463         if (!opts->sample_id_all_missing &&
 464             (opts->sample_time || !opts->no_inherit ||
 465              perf_target__has_cpu(&opts->target)))
 466                 attr->sample_type       |= PERF_SAMPLE_TIME;
 467
 468         if (opts->raw_samples) {
 469                 attr->sample_type       |= PERF_SAMPLE_TIME;
 470                 attr->sample_type       |= PERF_SAMPLE_RAW;
 471                 attr->sample_type       |= PERF_SAMPLE_CPU;
 472         }
 473
 474         if (opts->no_delay) {
 475                 attr->watermark = 0;
 476                 attr->wakeup_events = 1;
 477         }
 478         if (opts->branch_stack) {
 479                 attr->sample_type       |= PERF_SAMPLE_BRANCH_STACK;
 480                 attr->branch_sample_type = opts->branch_stack;
 481         }
 482
 483         attr->mmap = track;
 484         attr->comm = track;
 485
 486         if (perf_target__none(&opts->target) &&
 487             (!opts->group || evsel == first)) {
 488                 attr->enable_on_exec = 1;
 489         }
 490 }
 491
 492 int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
 493 {
 494         int cpu, thread;
 495         evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
 496
 497         if (evsel->fd) {
 498                 for (cpu = 0; cpu < ncpus; cpu++) {
 499                         for (thread = 0; thread < nthreads; thread++) {
 500                                 FD(evsel, cpu, thread) = -1;
 501                         }
 502                 }
 503         }
 504
 505         return evsel->fd != NULL ? 0 : -ENOMEM;
 506 }
 507
 508 int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
 509 {
 510         evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
 511         if (evsel->sample_id == NULL)
 512                 return -ENOMEM;
 513
 514         evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
 515         if (evsel->id == NULL) {
 516                 xyarray__delete(evsel->sample_id);
 517                 evsel->sample_id = NULL;
 518                 return -ENOMEM;
 519         }
 520
 521         return 0;
 522 }
 523
 524 int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus)
 525 {
 526         evsel->counts = zalloc((sizeof(*evsel->counts) +
 527                                 (ncpus * sizeof(struct perf_counts_values))));
 528         return evsel->counts != NULL ? 0 : -ENOMEM;
 529 }
 530
 531 void perf_evsel__free_fd(struct perf_evsel *evsel)
 532 {
 533         xyarray__delete(evsel->fd);
 534         evsel->fd = NULL;
 535 }
 536
 537 void perf_evsel__free_id(struct perf_evsel *evsel)
 538 {
 539         xyarray__delete(evsel->sample_id);
 540         evsel->sample_id = NULL;
 541         free(evsel->id);
 542         evsel->id = NULL;
 543 }
 544
 545 void perf_evsel__close_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
 546 {
 547         int cpu, thread;
 548
 549         for (cpu = 0; cpu < ncpus; cpu++)
 550                 for (thread = 0; thread < nthreads; ++thread) {
 551                         close(FD(evsel, cpu, thread));
 552                         FD(evsel, cpu, thread) = -1;
 553                 }
 554 }
 555
 556 void perf_evsel__exit(struct perf_evsel *evsel)
 557 {
 558         assert(list_empty(&evsel->node));
 559         xyarray__delete(evsel->fd);
 560         xyarray__delete(evsel->sample_id);
 561         free(evsel->id);
 562 }
 563
 564 void perf_evsel__delete(struct perf_evsel *evsel)
 565 {
 566         perf_evsel__exit(evsel);
 567         close_cgroup(evsel->cgrp);
 568         free(evsel->group_name);
 569         if (evsel->tp_format && evsel->name == evsel->tp_format->name) {
 570                 evsel->name = NULL;
 571                 pevent_free_format(evsel->tp_format);
 572         }
 573         free(evsel->name);
 574         free(evsel);
 575 }
 576
 577 int __perf_evsel__read_on_cpu(struct perf_evsel *evsel,
 578                               int cpu, int thread, bool scale)
 579 {
 580         struct perf_counts_values count;
 581         size_t nv = scale ? 3 : 1;
 582
 583         if (FD(evsel, cpu, thread) < 0)
 584                 return -EINVAL;
 585
 586         if (evsel->counts == NULL && perf_evsel__alloc_counts(evsel, cpu + 1) < 0)
 587                 return -ENOMEM;
 588
 589         if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) < 0)
 590                 return -errno;
 591
 592         if (scale) {
 593                 if (count.run == 0)
 594                         count.val = 0;
 595                 else if (count.run < count.ena)
 596                         count.val = (u64)((double)count.val * count.ena / count.run + 0.5);
 597         } else
 598                 count.ena = count.run = 0;
 599
 600         evsel->counts->cpu[cpu] = count;
 601         return 0;
 602 }
 603
 604 int __perf_evsel__read(struct perf_evsel *evsel,
 605                        int ncpus, int nthreads, bool scale)
 606 {
 607         size_t nv = scale ? 3 : 1;
 608         int cpu, thread;
 609         struct perf_counts_values *aggr = &evsel->counts->aggr, count;
 610
 611         aggr->val = aggr->ena = aggr->run = 0;
 612
 613         for (cpu = 0; cpu < ncpus; cpu++) {
 614                 for (thread = 0; thread < nthreads; thread++) {
 615                         if (FD(evsel, cpu, thread) < 0)
 616                                 continue;
 617
 618                         if (readn(FD(evsel, cpu, thread),
 619                                   &count, nv * sizeof(u64)) < 0)
 620                                 return -errno;
 621
 622                         aggr->val += count.val;
 623                         if (scale) {
 624                                 aggr->ena += count.ena;
 625                                 aggr->run += count.run;
 626                         }
 627                 }
 628         }
 629
 630         evsel->counts->scaled = 0;
 631         if (scale) {
 632                 if (aggr->run == 0) {
 633                         evsel->counts->scaled = -1;
 634                         aggr->val = 0;
 635                         return 0;
 636                 }
 637
 638                 if (aggr->run < aggr->ena) {
 639                         evsel->counts->scaled = 1;
 640                         aggr->val = (u64)((double)aggr->val * aggr->ena / aggr->run + 0.5);
 641                 }
 642         } else
 643                 aggr->ena = aggr->run = 0;
 644
 645         return 0;
 646 }
 647
 648 static int get_group_fd(struct perf_evsel *evsel, int cpu, int thread)
 649 {
 650         struct perf_evsel *leader = evsel->leader;
 651         int fd;
 652
 653         if (!leader)
 654                 return -1;
 655
 656         /*
 657          * Leader must be already processed/open,
 658          * if not it's a bug.
 659          */
 660         BUG_ON(!leader->fd);
 661
 662         fd = FD(leader, cpu, thread);
 663         BUG_ON(fd == -1);
 664
 665         return fd;
 666 }
 667
 668 static int __perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
 669                               struct thread_map *threads)
 670 {
 671         int cpu, thread;
 672         unsigned long flags = 0;
 673         int pid = -1, err;
 674
 675         if (evsel->fd == NULL &&
 676             perf_evsel__alloc_fd(evsel, cpus->nr, threads->nr) < 0)
 677                 return -ENOMEM;
 678
 679         if (evsel->cgrp) {
 680                 flags = PERF_FLAG_PID_CGROUP;
 681                 pid = evsel->cgrp->fd;
 682         }
 683
 684         for (cpu = 0; cpu < cpus->nr; cpu++) {
 685
 686                 for (thread = 0; thread < threads->nr; thread++) {
 687                         int group_fd;
 688
 689                         if (!evsel->cgrp)
 690                                 pid = threads->map[thread];
 691
 692                         group_fd = get_group_fd(evsel, cpu, thread);
 693
 694                         FD(evsel, cpu, thread) = sys_perf_event_open(&evsel->attr,
 695                                                                      pid,
 696                                                                      cpus->map[cpu],
 697                                                                      group_fd, flags);
 698                         if (FD(evsel, cpu, thread) < 0) {
 699                                 err = -errno;
 700                                 goto out_close;
 701                         }
 702                 }
 703         }
 704
 705         return 0;
 706
 707 out_close:
 708         do {
 709                 while (--thread >= 0) {
 710                         close(FD(evsel, cpu, thread));
 711                         FD(evsel, cpu, thread) = -1;
 712                 }
 713                 thread = threads->nr;
 714         } while (--cpu >= 0);
 715         return err;
 716 }
 717
 718 void perf_evsel__close(struct perf_evsel *evsel, int ncpus, int nthreads)
 719 {
 720         if (evsel->fd == NULL)
 721                 return;
 722
 723         perf_evsel__close_fd(evsel, ncpus, nthreads);
 724         perf_evsel__free_fd(evsel);
 725         evsel->fd = NULL;
 726 }
 727
 728 static struct {
 729         struct cpu_map map;
 730         int cpus[1];
 731 } empty_cpu_map = {
 732         .map.nr = 1,
 733         .cpus   = { -1, },
 734 };
 735
 736 static struct {
 737         struct thread_map map;
 738         int threads[1];
 739 } empty_thread_map = {
 740         .map.nr  = 1,
 741         .threads = { -1, },
 742 };
 743
 744 int perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
 745                      struct thread_map *threads)
 746 {
 747         if (cpus == NULL) {
 748                 /* Work around old compiler warnings about strict aliasing */
 749                 cpus = &empty_cpu_map.map;
 750         }
 751
 752         if (threads == NULL)
 753                 threads = &empty_thread_map.map;
 754
 755         return __perf_evsel__open(evsel, cpus, threads);
 756 }
 757
 758 int perf_evsel__open_per_cpu(struct perf_evsel *evsel,
 759                              struct cpu_map *cpus)
 760 {
 761         return __perf_evsel__open(evsel, cpus, &empty_thread_map.map);
 762 }
 763
 764 int perf_evsel__open_per_thread(struct perf_evsel *evsel,
 765                                 struct thread_map *threads)
 766 {
 767         return __perf_evsel__open(evsel, &empty_cpu_map.map, threads);
 768 }
 769
 770 static int perf_evsel__parse_id_sample(const struct perf_evsel *evsel,
 771                                        const union perf_event *event,
 772                                        struct perf_sample *sample)
 773 {
 774         u64 type = evsel->attr.sample_type;
 775         const u64 *array = event->sample.array;
 776         bool swapped = evsel->needs_swap;
 777         union u64_swap u;
 778
 779         array += ((event->header.size -
 780                    sizeof(event->header)) / sizeof(u64)) - 1;
 781
 782         if (type & PERF_SAMPLE_CPU) {
 783                 u.val64 = *array;
 784                 if (swapped) {
 785                         /* undo swap of u64, then swap on individual u32s */
 786                         u.val64 = bswap_64(u.val64);
 787                         u.val32[0] = bswap_32(u.val32[0]);
 788                 }
 789
 790                 sample->cpu = u.val32[0];
 791                 array--;
 792         }
 793
 794         if (type & PERF_SAMPLE_STREAM_ID) {
 795                 sample->stream_id = *array;
 796                 array--;
 797         }
 798
 799         if (type & PERF_SAMPLE_ID) {
 800                 sample->id = *array;
 801                 array--;
 802         }
 803
 804         if (type & PERF_SAMPLE_TIME) {
 805                 sample->time = *array;
 806                 array--;
 807         }
 808
 809         if (type & PERF_SAMPLE_TID) {
 810                 u.val64 = *array;
 811                 if (swapped) {
 812                         /* undo swap of u64, then swap on individual u32s */
 813                         u.val64 = bswap_64(u.val64);
 814                         u.val32[0] = bswap_32(u.val32[0]);
 815                         u.val32[1] = bswap_32(u.val32[1]);
 816                 }
 817
 818                 sample->pid = u.val32[0];
 819                 sample->tid = u.val32[1];
 820         }
 821
 822         return 0;
 823 }
 824
 825 static bool sample_overlap(const union perf_event *event,
 826                            const void *offset, u64 size)
 827 {
 828         const void *base = event;
 829
 830         if (offset + size > base + event->header.size)
 831                 return true;
 832
 833         return false;
 834 }
 835
 836 int perf_evsel__parse_sample(struct perf_evsel *evsel, union perf_event *event,
 837                              struct perf_sample *data)
 838 {
 839         u64 type = evsel->attr.sample_type;
 840         u64 regs_user = evsel->attr.sample_regs_user;
 841         bool swapped = evsel->needs_swap;
 842         const u64 *array;
 843
 844         /*
 845          * used for cross-endian analysis. See git commit 65014ab3
 846          * for why this goofiness is needed.
 847          */
 848         union u64_swap u;
 849
 850         memset(data, 0, sizeof(*data));
 851         data->cpu = data->pid = data->tid = -1;
 852         data->stream_id = data->id = data->time = -1ULL;
 853         data->period = 1;
 854
 855         if (event->header.type != PERF_RECORD_SAMPLE) {
 856                 if (!evsel->attr.sample_id_all)
 857                         return 0;
 858                 return perf_evsel__parse_id_sample(evsel, event, data);
 859         }
 860
 861         array = event->sample.array;
 862
 863         if (evsel->sample_size + sizeof(event->header) > event->header.size)
 864                 return -EFAULT;
 865
 866         if (type & PERF_SAMPLE_IP) {
 867                 data->ip = event->ip.ip;
 868                 array++;
 869         }
 870
 871         if (type & PERF_SAMPLE_TID) {
 872                 u.val64 = *array;
 873                 if (swapped) {
 874                         /* undo swap of u64, then swap on individual u32s */
 875                         u.val64 = bswap_64(u.val64);
 876                         u.val32[0] = bswap_32(u.val32[0]);
 877                         u.val32[1] = bswap_32(u.val32[1]);
 878                 }
 879
 880                 data->pid = u.val32[0];
 881                 data->tid = u.val32[1];
 882                 array++;
 883         }
 884
 885         if (type & PERF_SAMPLE_TIME) {
 886                 data->time = *array;
 887                 array++;
 888         }
 889
 890         data->addr = 0;
 891         if (type & PERF_SAMPLE_ADDR) {
 892                 data->addr = *array;
 893                 array++;
 894         }
 895
 896         data->id = -1ULL;
 897         if (type & PERF_SAMPLE_ID) {
 898                 data->id = *array;
 899                 array++;
 900         }
 901
 902         if (type & PERF_SAMPLE_STREAM_ID) {
 903                 data->stream_id = *array;
 904                 array++;
 905         }
 906
 907         if (type & PERF_SAMPLE_CPU) {
 908
 909                 u.val64 = *array;
 910                 if (swapped) {
 911                         /* undo swap of u64, then swap on individual u32s */
 912                         u.val64 = bswap_64(u.val64);
 913                         u.val32[0] = bswap_32(u.val32[0]);
 914                 }
 915
 916                 data->cpu = u.val32[0];
 917                 array++;
 918         }
 919
 920         if (type & PERF_SAMPLE_PERIOD) {
 921                 data->period = *array;
 922                 array++;
 923         }
 924
 925         if (type & PERF_SAMPLE_READ) {
 926                 fprintf(stderr, "PERF_SAMPLE_READ is unsupported for now\n");
 927                 return -1;
 928         }
 929
 930         if (type & PERF_SAMPLE_CALLCHAIN) {
 931                 if (sample_overlap(event, array, sizeof(data->callchain->nr)))
 932                         return -EFAULT;
 933
 934                 data->callchain = (struct ip_callchain *)array;
 935
 936                 if (sample_overlap(event, array, data->callchain->nr))
 937                         return -EFAULT;
 938
 939                 array += 1 + data->callchain->nr;
 940         }
 941
 942         if (type & PERF_SAMPLE_RAW) {
 943                 const u64 *pdata;
 944
 945                 u.val64 = *array;
 946                 if (WARN_ONCE(swapped,
 947                               "Endianness of raw data not corrected!\n")) {
 948                         /* undo swap of u64, then swap on individual u32s */
 949                         u.val64 = bswap_64(u.val64);
 950                         u.val32[0] = bswap_32(u.val32[0]);
 951                         u.val32[1] = bswap_32(u.val32[1]);
 952                 }
 953
 954                 if (sample_overlap(event, array, sizeof(u32)))
 955                         return -EFAULT;
 956
 957                 data->raw_size = u.val32[0];
 958                 pdata = (void *) array + sizeof(u32);
 959
 960                 if (sample_overlap(event, pdata, data->raw_size))
 961                         return -EFAULT;
 962
 963                 data->raw_data = (void *) pdata;
 964
 965                 array = (void *)array + data->raw_size + sizeof(u32);
 966         }
 967
 968         if (type & PERF_SAMPLE_BRANCH_STACK) {
 969                 u64 sz;
 970
 971                 data->branch_stack = (struct branch_stack *)array;
 972                 array++; /* nr */
 973
 974                 sz = data->branch_stack->nr * sizeof(struct branch_entry);
 975                 sz /= sizeof(u64);
 976                 array += sz;
 977         }
 978
 979         if (type & PERF_SAMPLE_REGS_USER) {
 980                 /* First u64 tells us if we have any regs in sample. */
 981                 u64 avail = *array++;
 982
 983                 if (avail) {
 984                         data->user_regs.regs = (u64 *)array;
 985                         array += hweight_long(regs_user);
 986                 }
 987         }
 988
 989         if (type & PERF_SAMPLE_STACK_USER) {
 990                 u64 size = *array++;
 991
 992                 data->user_stack.offset = ((char *)(array - 1)
 993                                           - (char *) event);
 994
 995                 if (!size) {
 996                         data->user_stack.size = 0;
 997                 } else {
 998                         data->user_stack.data = (char *)array;
 999                         array += size / sizeof(*array);
1000                         data->user_stack.size = *array;
1001                 }
1002         }
1003
1004         return 0;
1005 }
1006
1007 int perf_event__synthesize_sample(union perf_event *event, u64 type,
1008                                   const struct perf_sample *sample,
1009                                   bool swapped)
1010 {
1011         u64 *array;
1012
1013         /*
1014          * used for cross-endian analysis. See git commit 65014ab3
1015          * for why this goofiness is needed.
1016          */
1017         union u64_swap u;
1018
1019         array = event->sample.array;
1020
1021         if (type & PERF_SAMPLE_IP) {
1022                 event->ip.ip = sample->ip;
1023                 array++;
1024         }
1025
1026         if (type & PERF_SAMPLE_TID) {
1027                 u.val32[0] = sample->pid;
1028                 u.val32[1] = sample->tid;
1029                 if (swapped) {
1030                         /*
1031                          * Inverse of what is done in perf_evsel__parse_sample
1032                          */
1033                         u.val32[0] = bswap_32(u.val32[0]);
1034                         u.val32[1] = bswap_32(u.val32[1]);
1035                         u.val64 = bswap_64(u.val64);
1036                 }
1037
1038                 *array = u.val64;
1039                 array++;
1040         }
1041
1042         if (type & PERF_SAMPLE_TIME) {
1043                 *array = sample->time;
1044                 array++;
1045         }
1046
1047         if (type & PERF_SAMPLE_ADDR) {
1048                 *array = sample->addr;
1049                 array++;
1050         }
1051
1052         if (type & PERF_SAMPLE_ID) {
1053                 *array = sample->id;
1054                 array++;
1055         }
1056
1057         if (type & PERF_SAMPLE_STREAM_ID) {
1058                 *array = sample->stream_id;
1059                 array++;
1060         }
1061
1062         if (type & PERF_SAMPLE_CPU) {
1063                 u.val32[0] = sample->cpu;
1064                 if (swapped) {
1065                         /*
1066                          * Inverse of what is done in perf_evsel__parse_sample
1067                          */
1068                         u.val32[0] = bswap_32(u.val32[0]);
1069                         u.val64 = bswap_64(u.val64);
1070                 }
1071                 *array = u.val64;
1072                 array++;
1073         }
1074
1075         if (type & PERF_SAMPLE_PERIOD) {
1076                 *array = sample->period;
1077                 array++;
1078         }
1079
1080         return 0;
1081 }
1082
1083 struct format_field *perf_evsel__field(struct perf_evsel *evsel, const char *name)
1084 {
1085         return pevent_find_field(evsel->tp_format, name);
1086 }
1087
1088 char *perf_evsel__strval(struct perf_evsel *evsel, struct perf_sample *sample,
1089                          const char *name)
1090 {
1091         struct format_field *field = perf_evsel__field(evsel, name);
1092         int offset;
1093
1094         if (!field)
1095                 return NULL;
1096
1097         offset = field->offset;
1098
1099         if (field->flags & FIELD_IS_DYNAMIC) {
1100                 offset = *(int *)(sample->raw_data + field->offset);
1101                 offset &= 0xffff;
1102         }
1103
1104         return sample->raw_data + offset;
1105 }
1106
1107 u64 perf_evsel__intval(struct perf_evsel *evsel, struct perf_sample *sample,
1108                        const char *name)
1109 {
1110         struct format_field *field = perf_evsel__field(evsel, name);
1111         void *ptr;
1112         u64 value;
1113
1114         if (!field)
1115                 return 0;
1116
1117         ptr = sample->raw_data + field->offset;
1118
1119         switch (field->size) {
1120         case 1:
1121                 return *(u8 *)ptr;
1122         case 2:
1123                 value = *(u16 *)ptr;
1124                 break;
1125         case 4:
1126                 value = *(u32 *)ptr;
1127                 break;
1128         case 8:
1129                 value = *(u64 *)ptr;
1130                 break;
1131         default:
1132                 return 0;
1133         }
1134
1135         if (!evsel->needs_swap)
1136                 return value;
1137
1138         switch (field->size) {
1139         case 2:
1140                 return bswap_16(value);
1141         case 4:
1142                 return bswap_32(value);
1143         case 8:
1144                 return bswap_64(value);
1145         default:
1146                 return 0;
1147         }
1148
1149         return 0;
1150 }