tools/perf/util/evsel.c

   1 /*
   2  * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
   3  *
   4  * Parts came from builtin-{top,stat,record}.c, see those files for further
   5  * copyright notes.
   6  *
   7  * Released under the GPL v2. (and only v2, not any later version)
   8  */
   9
  10 #include <byteswap.h>
  11 #include <linux/bitops.h>
  12 #include "asm/bug.h"
  13 #include "debugfs.h"
  14 #include "event-parse.h"
  15 #include "evsel.h"
  16 #include "evlist.h"
  17 #include "util.h"
  18 #include "cpumap.h"
  19 #include "thread_map.h"
  20 #include "target.h"
  21 #include "../../../include/linux/hw_breakpoint.h"
  22 #include "../../include/linux/perf_event.h"
  23 #include "perf_regs.h"
  24
  25 #define FD(e, x, y) (*(int *)xyarray__entry(e->fd, x, y))
  26
  27 static int __perf_evsel__sample_size(u64 sample_type)
  28 {
  29         u64 mask = sample_type & PERF_SAMPLE_MASK;
  30         int size = 0;
  31         int i;
  32
  33         for (i = 0; i < 64; i++) {
  34                 if (mask & (1ULL << i))
  35                         size++;
  36         }
  37
  38         size *= sizeof(u64);
  39
  40         return size;
  41 }
  42
  43 void hists__init(struct hists *hists)
  44 {
  45         memset(hists, 0, sizeof(*hists));
  46         hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT;
  47         hists->entries_in = &hists->entries_in_array[0];
  48         hists->entries_collapsed = RB_ROOT;
  49         hists->entries = RB_ROOT;
  50         pthread_mutex_init(&hists->lock, NULL);
  51 }
  52
  53 void perf_evsel__init(struct perf_evsel *evsel,
  54                       struct perf_event_attr *attr, int idx)
  55 {
  56         evsel->idx         = idx;
  57         evsel->attr        = *attr;
  58         INIT_LIST_HEAD(&evsel->node);
  59         hists__init(&evsel->hists);
  60         evsel->sample_size = __perf_evsel__sample_size(attr->sample_type);
  61 }
  62
  63 struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr, int idx)
  64 {
  65         struct perf_evsel *evsel = zalloc(sizeof(*evsel));
  66
  67         if (evsel != NULL)
  68                 perf_evsel__init(evsel, attr, idx);
  69
  70         return evsel;
  71 }
  72
  73 struct event_format *event_format__new(const char *sys, const char *name)
  74 {
  75         int fd, n;
  76         char *filename;
  77         void *bf = NULL, *nbf;
  78         size_t size = 0, alloc_size = 0;
  79         struct event_format *format = NULL;
  80
  81         if (asprintf(&filename, "%s/%s/%s/format", tracing_events_path, sys, name) < 0)
  82                 goto out;
  83
  84         fd = open(filename, O_RDONLY);
  85         if (fd < 0)
  86                 goto out_free_filename;
  87
  88         do {
  89                 if (size == alloc_size) {
  90                         alloc_size += BUFSIZ;
  91                         nbf = realloc(bf, alloc_size);
  92                         if (nbf == NULL)
  93                                 goto out_free_bf;
  94                         bf = nbf;
  95                 }
  96
  97                 n = read(fd, bf + size, BUFSIZ);
  98                 if (n < 0)
  99                         goto out_free_bf;
 100                 size += n;
 101         } while (n > 0);
 102
 103         pevent_parse_format(&format, bf, size, sys);
 104
 105 out_free_bf:
 106         free(bf);
 107         close(fd);
 108 out_free_filename:
 109         free(filename);
 110 out:
 111         return format;
 112 }
 113
 114 struct perf_evsel *perf_evsel__newtp(const char *sys, const char *name, int idx)
 115 {
 116         struct perf_evsel *evsel = zalloc(sizeof(*evsel));
 117
 118         if (evsel != NULL) {
 119                 struct perf_event_attr attr = {
 120                         .type          = PERF_TYPE_TRACEPOINT,
 121                         .sample_type   = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
 122                                           PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
 123                 };
 124
 125                 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
 126                         goto out_free;
 127
 128                 evsel->tp_format = event_format__new(sys, name);
 129                 if (evsel->tp_format == NULL)
 130                         goto out_free;
 131
 132                 event_attr_init(&attr);
 133                 attr.config = evsel->tp_format->id;
 134                 attr.sample_period = 1;
 135                 perf_evsel__init(evsel, &attr, idx);
 136         }
 137
 138         return evsel;
 139
 140 out_free:
 141         free(evsel->name);
 142         free(evsel);
 143         return NULL;
 144 }
 145
 146 const char *perf_evsel__hw_names[PERF_COUNT_HW_MAX] = {
 147         "cycles",
 148         "instructions",
 149         "cache-references",
 150         "cache-misses",
 151         "branches",
 152         "branch-misses",
 153         "bus-cycles",
 154         "stalled-cycles-frontend",
 155         "stalled-cycles-backend",
 156         "ref-cycles",
 157 };
 158
 159 static const char *__perf_evsel__hw_name(u64 config)
 160 {
 161         if (config < PERF_COUNT_HW_MAX && perf_evsel__hw_names[config])
 162                 return perf_evsel__hw_names[config];
 163
 164         return "unknown-hardware";
 165 }
 166
 167 static int perf_evsel__add_modifiers(struct perf_evsel *evsel, char *bf, size_t size)
 168 {
 169         int colon = 0, r = 0;
 170         struct perf_event_attr *attr = &evsel->attr;
 171         bool exclude_guest_default = false;
 172
 173 #define MOD_PRINT(context, mod) do {                                    \
 174                 if (!attr->exclude_##context) {                         \
 175                         if (!colon) colon = ++r;                        \
 176                         r += scnprintf(bf + r, size - r, "%c", mod);    \
 177                 } } while(0)
 178
 179         if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
 180                 MOD_PRINT(kernel, 'k');
 181                 MOD_PRINT(user, 'u');
 182                 MOD_PRINT(hv, 'h');
 183                 exclude_guest_default = true;
 184         }
 185
 186         if (attr->precise_ip) {
 187                 if (!colon)
 188                         colon = ++r;
 189                 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
 190                 exclude_guest_default = true;
 191         }
 192
 193         if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
 194                 MOD_PRINT(host, 'H');
 195                 MOD_PRINT(guest, 'G');
 196         }
 197 #undef MOD_PRINT
 198         if (colon)
 199                 bf[colon - 1] = ':';
 200         return r;
 201 }
 202
 203 static int perf_evsel__hw_name(struct perf_evsel *evsel, char *bf, size_t size)
 204 {
 205         int r = scnprintf(bf, size, "%s", __perf_evsel__hw_name(evsel->attr.config));
 206         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 207 }
 208
 209 const char *perf_evsel__sw_names[PERF_COUNT_SW_MAX] = {
 210         "cpu-clock",
 211         "task-clock",
 212         "page-faults",
 213         "context-switches",
 214         "cpu-migrations",
 215         "minor-faults",
 216         "major-faults",
 217         "alignment-faults",
 218         "emulation-faults",
 219 };
 220
 221 static const char *__perf_evsel__sw_name(u64 config)
 222 {
 223         if (config < PERF_COUNT_SW_MAX && perf_evsel__sw_names[config])
 224                 return perf_evsel__sw_names[config];
 225         return "unknown-software";
 226 }
 227
 228 static int perf_evsel__sw_name(struct perf_evsel *evsel, char *bf, size_t size)
 229 {
 230         int r = scnprintf(bf, size, "%s", __perf_evsel__sw_name(evsel->attr.config));
 231         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 232 }
 233
 234 static int __perf_evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
 235 {
 236         int r;
 237
 238         r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
 239
 240         if (type & HW_BREAKPOINT_R)
 241                 r += scnprintf(bf + r, size - r, "r");
 242
 243         if (type & HW_BREAKPOINT_W)
 244                 r += scnprintf(bf + r, size - r, "w");
 245
 246         if (type & HW_BREAKPOINT_X)
 247                 r += scnprintf(bf + r, size - r, "x");
 248
 249         return r;
 250 }
 251
 252 static int perf_evsel__bp_name(struct perf_evsel *evsel, char *bf, size_t size)
 253 {
 254         struct perf_event_attr *attr = &evsel->attr;
 255         int r = __perf_evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
 256         return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
 257 }
 258
 259 const char *perf_evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX]
 260                                 [PERF_EVSEL__MAX_ALIASES] = {
 261  { "L1-dcache", "l1-d",         "l1d",          "L1-data",              },
 262  { "L1-icache", "l1-i",         "l1i",          "L1-instruction",       },
 263  { "LLC",       "L2",                                                   },
 264  { "dTLB",      "d-tlb",        "Data-TLB",                             },
 265  { "iTLB",      "i-tlb",        "Instruction-TLB",                      },
 266  { "branch",    "branches",     "bpu",          "btb",          "bpc",  },
 267  { "node",                                                              },
 268 };
 269
 270 const char *perf_evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX]
 271                                    [PERF_EVSEL__MAX_ALIASES] = {
 272  { "load",      "loads",        "read",                                 },
 273  { "store",     "stores",       "write",                                },
 274  { "prefetch",  "prefetches",   "speculative-read", "speculative-load", },
 275 };
 276
 277 const char *perf_evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX]
 278                                        [PERF_EVSEL__MAX_ALIASES] = {
 279  { "refs",      "Reference",    "ops",          "access",               },
 280  { "misses",    "miss",                                                 },
 281 };
 282
 283 #define C(x)            PERF_COUNT_HW_CACHE_##x
 284 #define CACHE_READ      (1 << C(OP_READ))
 285 #define CACHE_WRITE     (1 << C(OP_WRITE))
 286 #define CACHE_PREFETCH  (1 << C(OP_PREFETCH))
 287 #define COP(x)          (1 << x)
 288
 289 /*
 290  * cache operartion stat
 291  * L1I : Read and prefetch only
 292  * ITLB and BPU : Read-only
 293  */
 294 static unsigned long perf_evsel__hw_cache_stat[C(MAX)] = {
 295  [C(L1D)]       = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 296  [C(L1I)]       = (CACHE_READ | CACHE_PREFETCH),
 297  [C(LL)]        = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 298  [C(DTLB)]      = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 299  [C(ITLB)]      = (CACHE_READ),
 300  [C(BPU)]       = (CACHE_READ),
 301  [C(NODE)]      = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 302 };
 303
 304 bool perf_evsel__is_cache_op_valid(u8 type, u8 op)
 305 {
 306         if (perf_evsel__hw_cache_stat[type] & COP(op))
 307                 return true;    /* valid */
 308         else
 309                 return false;   /* invalid */
 310 }
 311
 312 int __perf_evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result,
 313                                             char *bf, size_t size)
 314 {
 315         if (result) {
 316                 return scnprintf(bf, size, "%s-%s-%s", perf_evsel__hw_cache[type][0],
 317                                  perf_evsel__hw_cache_op[op][0],
 318                                  perf_evsel__hw_cache_result[result][0]);
 319         }
 320
 321         return scnprintf(bf, size, "%s-%s", perf_evsel__hw_cache[type][0],
 322                          perf_evsel__hw_cache_op[op][1]);
 323 }
 324
 325 static int __perf_evsel__hw_cache_name(u64 config, char *bf, size_t size)
 326 {
 327         u8 op, result, type = (config >>  0) & 0xff;
 328         const char *err = "unknown-ext-hardware-cache-type";
 329
 330         if (type > PERF_COUNT_HW_CACHE_MAX)
 331                 goto out_err;
 332
 333         op = (config >>  8) & 0xff;
 334         err = "unknown-ext-hardware-cache-op";
 335         if (op > PERF_COUNT_HW_CACHE_OP_MAX)
 336                 goto out_err;
 337
 338         result = (config >> 16) & 0xff;
 339         err = "unknown-ext-hardware-cache-result";
 340         if (result > PERF_COUNT_HW_CACHE_RESULT_MAX)
 341                 goto out_err;
 342
 343         err = "invalid-cache";
 344         if (!perf_evsel__is_cache_op_valid(type, op))
 345                 goto out_err;
 346
 347         return __perf_evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
 348 out_err:
 349         return scnprintf(bf, size, "%s", err);
 350 }
 351
 352 static int perf_evsel__hw_cache_name(struct perf_evsel *evsel, char *bf, size_t size)
 353 {
 354         int ret = __perf_evsel__hw_cache_name(evsel->attr.config, bf, size);
 355         return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
 356 }
 357
 358 static int perf_evsel__raw_name(struct perf_evsel *evsel, char *bf, size_t size)
 359 {
 360         int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->attr.config);
 361         return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
 362 }
 363
 364 const char *perf_evsel__name(struct perf_evsel *evsel)
 365 {
 366         char bf[128];
 367
 368         if (evsel->name)
 369                 return evsel->name;
 370
 371         switch (evsel->attr.type) {
 372         case PERF_TYPE_RAW:
 373                 perf_evsel__raw_name(evsel, bf, sizeof(bf));
 374                 break;
 375
 376         case PERF_TYPE_HARDWARE:
 377                 perf_evsel__hw_name(evsel, bf, sizeof(bf));
 378                 break;
 379
 380         case PERF_TYPE_HW_CACHE:
 381                 perf_evsel__hw_cache_name(evsel, bf, sizeof(bf));
 382                 break;
 383
 384         case PERF_TYPE_SOFTWARE:
 385                 perf_evsel__sw_name(evsel, bf, sizeof(bf));
 386                 break;
 387
 388         case PERF_TYPE_TRACEPOINT:
 389                 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
 390                 break;
 391
 392         case PERF_TYPE_BREAKPOINT:
 393                 perf_evsel__bp_name(evsel, bf, sizeof(bf));
 394                 break;
 395
 396         default:
 397                 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
 398                           evsel->attr.type);
 399                 break;
 400         }
 401
 402         evsel->name = strdup(bf);
 403
 404         return evsel->name ?: "unknown";
 405 }
 406
 407 void perf_evsel__config(struct perf_evsel *evsel, struct perf_record_opts *opts,
 408                         struct perf_evsel *first)
 409 {
 410         struct perf_event_attr *attr = &evsel->attr;
 411         int track = !evsel->idx; /* only the first counter needs these */
 412
 413         attr->disabled = 1;
 414         attr->sample_id_all = opts->sample_id_all_missing ? 0 : 1;
 415         attr->inherit       = !opts->no_inherit;
 416         attr->read_format   = PERF_FORMAT_TOTAL_TIME_ENABLED |
 417                               PERF_FORMAT_TOTAL_TIME_RUNNING |
 418                               PERF_FORMAT_ID;
 419
 420         attr->sample_type  |= PERF_SAMPLE_IP | PERF_SAMPLE_TID;
 421
 422         /*
 423          * We default some events to a 1 default interval. But keep
 424          * it a weak assumption overridable by the user.
 425          */
 426         if (!attr->sample_period || (opts->user_freq != UINT_MAX &&
 427                                      opts->user_interval != ULLONG_MAX)) {
 428                 if (opts->freq) {
 429                         attr->sample_type       |= PERF_SAMPLE_PERIOD;
 430                         attr->freq              = 1;
 431                         attr->sample_freq       = opts->freq;
 432                 } else {
 433                         attr->sample_period = opts->default_interval;
 434                 }
 435         }
 436
 437         if (opts->no_samples)
 438                 attr->sample_freq = 0;
 439
 440         if (opts->inherit_stat)
 441                 attr->inherit_stat = 1;
 442
 443         if (opts->sample_address) {
 444                 attr->sample_type       |= PERF_SAMPLE_ADDR;
 445                 attr->mmap_data = track;
 446         }
 447
 448         if (opts->call_graph) {
 449                 attr->sample_type       |= PERF_SAMPLE_CALLCHAIN;
 450
 451                 if (opts->call_graph == CALLCHAIN_DWARF) {
 452                         attr->sample_type |= PERF_SAMPLE_REGS_USER |
 453                                              PERF_SAMPLE_STACK_USER;
 454                         attr->sample_regs_user = PERF_REGS_MASK;
 455                         attr->sample_stack_user = opts->stack_dump_size;
 456                         attr->exclude_callchain_user = 1;
 457                 }
 458         }
 459
 460         if (perf_target__has_cpu(&opts->target))
 461                 attr->sample_type       |= PERF_SAMPLE_CPU;
 462
 463         if (opts->period)
 464                 attr->sample_type       |= PERF_SAMPLE_PERIOD;
 465
 466         if (!opts->sample_id_all_missing &&
 467             (opts->sample_time || !opts->no_inherit ||
 468              perf_target__has_cpu(&opts->target)))
 469                 attr->sample_type       |= PERF_SAMPLE_TIME;
 470
 471         if (opts->raw_samples) {
 472                 attr->sample_type       |= PERF_SAMPLE_TIME;
 473                 attr->sample_type       |= PERF_SAMPLE_RAW;
 474                 attr->sample_type       |= PERF_SAMPLE_CPU;
 475         }
 476
 477         if (opts->no_delay) {
 478                 attr->watermark = 0;
 479                 attr->wakeup_events = 1;
 480         }
 481         if (opts->branch_stack) {
 482                 attr->sample_type       |= PERF_SAMPLE_BRANCH_STACK;
 483                 attr->branch_sample_type = opts->branch_stack;
 484         }
 485
 486         attr->mmap = track;
 487         attr->comm = track;
 488
 489         if (perf_target__none(&opts->target) &&
 490             (!opts->group || evsel == first)) {
 491                 attr->enable_on_exec = 1;
 492         }
 493 }
 494
 495 int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
 496 {
 497         int cpu, thread;
 498         evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
 499
 500         if (evsel->fd) {
 501                 for (cpu = 0; cpu < ncpus; cpu++) {
 502                         for (thread = 0; thread < nthreads; thread++) {
 503                                 FD(evsel, cpu, thread) = -1;
 504                         }
 505                 }
 506         }
 507
 508         return evsel->fd != NULL ? 0 : -ENOMEM;
 509 }
 510
 511 int perf_evsel__set_filter(struct perf_evsel *evsel, int ncpus, int nthreads,
 512                            const char *filter)
 513 {
 514         int cpu, thread;
 515
 516         for (cpu = 0; cpu < ncpus; cpu++) {
 517                 for (thread = 0; thread < nthreads; thread++) {
 518                         int fd = FD(evsel, cpu, thread),
 519                             err = ioctl(fd, PERF_EVENT_IOC_SET_FILTER, filter);
 520
 521                         if (err)
 522                                 return err;
 523                 }
 524         }
 525
 526         return 0;
 527 }
 528
 529 int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
 530 {
 531         evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
 532         if (evsel->sample_id == NULL)
 533                 return -ENOMEM;
 534
 535         evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
 536         if (evsel->id == NULL) {
 537                 xyarray__delete(evsel->sample_id);
 538                 evsel->sample_id = NULL;
 539                 return -ENOMEM;
 540         }
 541
 542         return 0;
 543 }
 544
 545 int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus)
 546 {
 547         evsel->counts = zalloc((sizeof(*evsel->counts) +
 548                                 (ncpus * sizeof(struct perf_counts_values))));
 549         return evsel->counts != NULL ? 0 : -ENOMEM;
 550 }
 551
 552 void perf_evsel__free_fd(struct perf_evsel *evsel)
 553 {
 554         xyarray__delete(evsel->fd);
 555         evsel->fd = NULL;
 556 }
 557
 558 void perf_evsel__free_id(struct perf_evsel *evsel)
 559 {
 560         xyarray__delete(evsel->sample_id);
 561         evsel->sample_id = NULL;
 562         free(evsel->id);
 563         evsel->id = NULL;
 564 }
 565
 566 void perf_evsel__close_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
 567 {
 568         int cpu, thread;
 569
 570         for (cpu = 0; cpu < ncpus; cpu++)
 571                 for (thread = 0; thread < nthreads; ++thread) {
 572                         close(FD(evsel, cpu, thread));
 573                         FD(evsel, cpu, thread) = -1;
 574                 }
 575 }
 576
 577 void perf_evsel__exit(struct perf_evsel *evsel)
 578 {
 579         assert(list_empty(&evsel->node));
 580         xyarray__delete(evsel->fd);
 581         xyarray__delete(evsel->sample_id);
 582         free(evsel->id);
 583 }
 584
 585 void perf_evsel__delete(struct perf_evsel *evsel)
 586 {
 587         perf_evsel__exit(evsel);
 588         close_cgroup(evsel->cgrp);
 589         free(evsel->group_name);
 590         if (evsel->tp_format)
 591                 pevent_free_format(evsel->tp_format);
 592         free(evsel->name);
 593         free(evsel);
 594 }
 595
 596 int __perf_evsel__read_on_cpu(struct perf_evsel *evsel,
 597                               int cpu, int thread, bool scale)
 598 {
 599         struct perf_counts_values count;
 600         size_t nv = scale ? 3 : 1;
 601
 602         if (FD(evsel, cpu, thread) < 0)
 603                 return -EINVAL;
 604
 605         if (evsel->counts == NULL && perf_evsel__alloc_counts(evsel, cpu + 1) < 0)
 606                 return -ENOMEM;
 607
 608         if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) < 0)
 609                 return -errno;
 610
 611         if (scale) {
 612                 if (count.run == 0)
 613                         count.val = 0;
 614                 else if (count.run < count.ena)
 615                         count.val = (u64)((double)count.val * count.ena / count.run + 0.5);
 616         } else
 617                 count.ena = count.run = 0;
 618
 619         evsel->counts->cpu[cpu] = count;
 620         return 0;
 621 }
 622
 623 int __perf_evsel__read(struct perf_evsel *evsel,
 624                        int ncpus, int nthreads, bool scale)
 625 {
 626         size_t nv = scale ? 3 : 1;
 627         int cpu, thread;
 628         struct perf_counts_values *aggr = &evsel->counts->aggr, count;
 629
 630         aggr->val = aggr->ena = aggr->run = 0;
 631
 632         for (cpu = 0; cpu < ncpus; cpu++) {
 633                 for (thread = 0; thread < nthreads; thread++) {
 634                         if (FD(evsel, cpu, thread) < 0)
 635                                 continue;
 636
 637                         if (readn(FD(evsel, cpu, thread),
 638                                   &count, nv * sizeof(u64)) < 0)
 639                                 return -errno;
 640
 641                         aggr->val += count.val;
 642                         if (scale) {
 643                                 aggr->ena += count.ena;
 644                                 aggr->run += count.run;
 645                         }
 646                 }
 647         }
 648
 649         evsel->counts->scaled = 0;
 650         if (scale) {
 651                 if (aggr->run == 0) {
 652                         evsel->counts->scaled = -1;
 653                         aggr->val = 0;
 654                         return 0;
 655                 }
 656
 657                 if (aggr->run < aggr->ena) {
 658                         evsel->counts->scaled = 1;
 659                         aggr->val = (u64)((double)aggr->val * aggr->ena / aggr->run + 0.5);
 660                 }
 661         } else
 662                 aggr->ena = aggr->run = 0;
 663
 664         return 0;
 665 }
 666
 667 static int get_group_fd(struct perf_evsel *evsel, int cpu, int thread)
 668 {
 669         struct perf_evsel *leader = evsel->leader;
 670         int fd;
 671
 672         if (!leader)
 673                 return -1;
 674
 675         /*
 676          * Leader must be already processed/open,
 677          * if not it's a bug.
 678          */
 679         BUG_ON(!leader->fd);
 680
 681         fd = FD(leader, cpu, thread);
 682         BUG_ON(fd == -1);
 683
 684         return fd;
 685 }
 686
 687 static int __perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
 688                               struct thread_map *threads)
 689 {
 690         int cpu, thread;
 691         unsigned long flags = 0;
 692         int pid = -1, err;
 693
 694         if (evsel->fd == NULL &&
 695             perf_evsel__alloc_fd(evsel, cpus->nr, threads->nr) < 0)
 696                 return -ENOMEM;
 697
 698         if (evsel->cgrp) {
 699                 flags = PERF_FLAG_PID_CGROUP;
 700                 pid = evsel->cgrp->fd;
 701         }
 702
 703         for (cpu = 0; cpu < cpus->nr; cpu++) {
 704
 705                 for (thread = 0; thread < threads->nr; thread++) {
 706                         int group_fd;
 707
 708                         if (!evsel->cgrp)
 709                                 pid = threads->map[thread];
 710
 711                         group_fd = get_group_fd(evsel, cpu, thread);
 712
 713                         FD(evsel, cpu, thread) = sys_perf_event_open(&evsel->attr,
 714                                                                      pid,
 715                                                                      cpus->map[cpu],
 716                                                                      group_fd, flags);
 717                         if (FD(evsel, cpu, thread) < 0) {
 718                                 err = -errno;
 719                                 goto out_close;
 720                         }
 721                 }
 722         }
 723
 724         return 0;
 725
 726 out_close:
 727         do {
 728                 while (--thread >= 0) {
 729                         close(FD(evsel, cpu, thread));
 730                         FD(evsel, cpu, thread) = -1;
 731                 }
 732                 thread = threads->nr;
 733         } while (--cpu >= 0);
 734         return err;
 735 }
 736
 737 void perf_evsel__close(struct perf_evsel *evsel, int ncpus, int nthreads)
 738 {
 739         if (evsel->fd == NULL)
 740                 return;
 741
 742         perf_evsel__close_fd(evsel, ncpus, nthreads);
 743         perf_evsel__free_fd(evsel);
 744         evsel->fd = NULL;
 745 }
 746
 747 static struct {
 748         struct cpu_map map;
 749         int cpus[1];
 750 } empty_cpu_map = {
 751         .map.nr = 1,
 752         .cpus   = { -1, },
 753 };
 754
 755 static struct {
 756         struct thread_map map;
 757         int threads[1];
 758 } empty_thread_map = {
 759         .map.nr  = 1,
 760         .threads = { -1, },
 761 };
 762
 763 int perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
 764                      struct thread_map *threads)
 765 {
 766         if (cpus == NULL) {
 767                 /* Work around old compiler warnings about strict aliasing */
 768                 cpus = &empty_cpu_map.map;
 769         }
 770
 771         if (threads == NULL)
 772                 threads = &empty_thread_map.map;
 773
 774         return __perf_evsel__open(evsel, cpus, threads);
 775 }
 776
 777 int perf_evsel__open_per_cpu(struct perf_evsel *evsel,
 778                              struct cpu_map *cpus)
 779 {
 780         return __perf_evsel__open(evsel, cpus, &empty_thread_map.map);
 781 }
 782
 783 int perf_evsel__open_per_thread(struct perf_evsel *evsel,
 784                                 struct thread_map *threads)
 785 {
 786         return __perf_evsel__open(evsel, &empty_cpu_map.map, threads);
 787 }
 788
 789 static int perf_evsel__parse_id_sample(const struct perf_evsel *evsel,
 790                                        const union perf_event *event,
 791                                        struct perf_sample *sample)
 792 {
 793         u64 type = evsel->attr.sample_type;
 794         const u64 *array = event->sample.array;
 795         bool swapped = evsel->needs_swap;
 796         union u64_swap u;
 797
 798         array += ((event->header.size -
 799                    sizeof(event->header)) / sizeof(u64)) - 1;
 800
 801         if (type & PERF_SAMPLE_CPU) {
 802                 u.val64 = *array;
 803                 if (swapped) {
 804                         /* undo swap of u64, then swap on individual u32s */
 805                         u.val64 = bswap_64(u.val64);
 806                         u.val32[0] = bswap_32(u.val32[0]);
 807                 }
 808
 809                 sample->cpu = u.val32[0];
 810                 array--;
 811         }
 812
 813         if (type & PERF_SAMPLE_STREAM_ID) {
 814                 sample->stream_id = *array;
 815                 array--;
 816         }
 817
 818         if (type & PERF_SAMPLE_ID) {
 819                 sample->id = *array;
 820                 array--;
 821         }
 822
 823         if (type & PERF_SAMPLE_TIME) {
 824                 sample->time = *array;
 825                 array--;
 826         }
 827
 828         if (type & PERF_SAMPLE_TID) {
 829                 u.val64 = *array;
 830                 if (swapped) {
 831                         /* undo swap of u64, then swap on individual u32s */
 832                         u.val64 = bswap_64(u.val64);
 833                         u.val32[0] = bswap_32(u.val32[0]);
 834                         u.val32[1] = bswap_32(u.val32[1]);
 835                 }
 836
 837                 sample->pid = u.val32[0];
 838                 sample->tid = u.val32[1];
 839         }
 840
 841         return 0;
 842 }
 843
 844 static bool sample_overlap(const union perf_event *event,
 845                            const void *offset, u64 size)
 846 {
 847         const void *base = event;
 848
 849         if (offset + size > base + event->header.size)
 850                 return true;
 851
 852         return false;
 853 }
 854
 855 int perf_evsel__parse_sample(struct perf_evsel *evsel, union perf_event *event,
 856                              struct perf_sample *data)
 857 {
 858         u64 type = evsel->attr.sample_type;
 859         u64 regs_user = evsel->attr.sample_regs_user;
 860         bool swapped = evsel->needs_swap;
 861         const u64 *array;
 862
 863         /*
 864          * used for cross-endian analysis. See git commit 65014ab3
 865          * for why this goofiness is needed.
 866          */
 867         union u64_swap u;
 868
 869         memset(data, 0, sizeof(*data));
 870         data->cpu = data->pid = data->tid = -1;
 871         data->stream_id = data->id = data->time = -1ULL;
 872         data->period = 1;
 873
 874         if (event->header.type != PERF_RECORD_SAMPLE) {
 875                 if (!evsel->attr.sample_id_all)
 876                         return 0;
 877                 return perf_evsel__parse_id_sample(evsel, event, data);
 878         }
 879
 880         array = event->sample.array;
 881
 882         if (evsel->sample_size + sizeof(event->header) > event->header.size)
 883                 return -EFAULT;
 884
 885         if (type & PERF_SAMPLE_IP) {
 886                 data->ip = event->ip.ip;
 887                 array++;
 888         }
 889
 890         if (type & PERF_SAMPLE_TID) {
 891                 u.val64 = *array;
 892                 if (swapped) {
 893                         /* undo swap of u64, then swap on individual u32s */
 894                         u.val64 = bswap_64(u.val64);
 895                         u.val32[0] = bswap_32(u.val32[0]);
 896                         u.val32[1] = bswap_32(u.val32[1]);
 897                 }
 898
 899                 data->pid = u.val32[0];
 900                 data->tid = u.val32[1];
 901                 array++;
 902         }
 903
 904         if (type & PERF_SAMPLE_TIME) {
 905                 data->time = *array;
 906                 array++;
 907         }
 908
 909         data->addr = 0;
 910         if (type & PERF_SAMPLE_ADDR) {
 911                 data->addr = *array;
 912                 array++;
 913         }
 914
 915         data->id = -1ULL;
 916         if (type & PERF_SAMPLE_ID) {
 917                 data->id = *array;
 918                 array++;
 919         }
 920
 921         if (type & PERF_SAMPLE_STREAM_ID) {
 922                 data->stream_id = *array;
 923                 array++;
 924         }
 925
 926         if (type & PERF_SAMPLE_CPU) {
 927
 928                 u.val64 = *array;
 929                 if (swapped) {
 930                         /* undo swap of u64, then swap on individual u32s */
 931                         u.val64 = bswap_64(u.val64);
 932                         u.val32[0] = bswap_32(u.val32[0]);
 933                 }
 934
 935                 data->cpu = u.val32[0];
 936                 array++;
 937         }
 938
 939         if (type & PERF_SAMPLE_PERIOD) {
 940                 data->period = *array;
 941                 array++;
 942         }
 943
 944         if (type & PERF_SAMPLE_READ) {
 945                 fprintf(stderr, "PERF_SAMPLE_READ is unsupported for now\n");
 946                 return -1;
 947         }
 948
 949         if (type & PERF_SAMPLE_CALLCHAIN) {
 950                 if (sample_overlap(event, array, sizeof(data->callchain->nr)))
 951                         return -EFAULT;
 952
 953                 data->callchain = (struct ip_callchain *)array;
 954
 955                 if (sample_overlap(event, array, data->callchain->nr))
 956                         return -EFAULT;
 957
 958                 array += 1 + data->callchain->nr;
 959         }
 960
 961         if (type & PERF_SAMPLE_RAW) {
 962                 const u64 *pdata;
 963
 964                 u.val64 = *array;
 965                 if (WARN_ONCE(swapped,
 966                               "Endianness of raw data not corrected!\n")) {
 967                         /* undo swap of u64, then swap on individual u32s */
 968                         u.val64 = bswap_64(u.val64);
 969                         u.val32[0] = bswap_32(u.val32[0]);
 970                         u.val32[1] = bswap_32(u.val32[1]);
 971                 }
 972
 973                 if (sample_overlap(event, array, sizeof(u32)))
 974                         return -EFAULT;
 975
 976                 data->raw_size = u.val32[0];
 977                 pdata = (void *) array + sizeof(u32);
 978
 979                 if (sample_overlap(event, pdata, data->raw_size))
 980                         return -EFAULT;
 981
 982                 data->raw_data = (void *) pdata;
 983
 984                 array = (void *)array + data->raw_size + sizeof(u32);
 985         }
 986
 987         if (type & PERF_SAMPLE_BRANCH_STACK) {
 988                 u64 sz;
 989
 990                 data->branch_stack = (struct branch_stack *)array;
 991                 array++; /* nr */
 992
 993                 sz = data->branch_stack->nr * sizeof(struct branch_entry);
 994                 sz /= sizeof(u64);
 995                 array += sz;
 996         }
 997
 998         if (type & PERF_SAMPLE_REGS_USER) {
 999                 /* First u64 tells us if we have any regs in sample. */
1000                 u64 avail = *array++;
1001
1002                 if (avail) {
1003                         data->user_regs.regs = (u64 *)array;
1004                         array += hweight_long(regs_user);
1005                 }
1006         }
1007
1008         if (type & PERF_SAMPLE_STACK_USER) {
1009                 u64 size = *array++;
1010
1011                 data->user_stack.offset = ((char *)(array - 1)
1012                                           - (char *) event);
1013
1014                 if (!size) {
1015                         data->user_stack.size = 0;
1016                 } else {
1017                         data->user_stack.data = (char *)array;
1018                         array += size / sizeof(*array);
1019                         data->user_stack.size = *array;
1020                 }
1021         }
1022
1023         return 0;
1024 }
1025
1026 int perf_event__synthesize_sample(union perf_event *event, u64 type,
1027                                   const struct perf_sample *sample,
1028                                   bool swapped)
1029 {
1030         u64 *array;
1031
1032         /*
1033          * used for cross-endian analysis. See git commit 65014ab3
1034          * for why this goofiness is needed.
1035          */
1036         union u64_swap u;
1037
1038         array = event->sample.array;
1039
1040         if (type & PERF_SAMPLE_IP) {
1041                 event->ip.ip = sample->ip;
1042                 array++;
1043         }
1044
1045         if (type & PERF_SAMPLE_TID) {
1046                 u.val32[0] = sample->pid;
1047                 u.val32[1] = sample->tid;
1048                 if (swapped) {
1049                         /*
1050                          * Inverse of what is done in perf_evsel__parse_sample
1051                          */
1052                         u.val32[0] = bswap_32(u.val32[0]);
1053                         u.val32[1] = bswap_32(u.val32[1]);
1054                         u.val64 = bswap_64(u.val64);
1055                 }
1056
1057                 *array = u.val64;
1058                 array++;
1059         }
1060
1061         if (type & PERF_SAMPLE_TIME) {
1062                 *array = sample->time;
1063                 array++;
1064         }
1065
1066         if (type & PERF_SAMPLE_ADDR) {
1067                 *array = sample->addr;
1068                 array++;
1069         }
1070
1071         if (type & PERF_SAMPLE_ID) {
1072                 *array = sample->id;
1073                 array++;
1074         }
1075
1076         if (type & PERF_SAMPLE_STREAM_ID) {
1077                 *array = sample->stream_id;
1078                 array++;
1079         }
1080
1081         if (type & PERF_SAMPLE_CPU) {
1082                 u.val32[0] = sample->cpu;
1083                 if (swapped) {
1084                         /*
1085                          * Inverse of what is done in perf_evsel__parse_sample
1086                          */
1087                         u.val32[0] = bswap_32(u.val32[0]);
1088                         u.val64 = bswap_64(u.val64);
1089                 }
1090                 *array = u.val64;
1091                 array++;
1092         }
1093
1094         if (type & PERF_SAMPLE_PERIOD) {
1095                 *array = sample->period;
1096                 array++;
1097         }
1098
1099         return 0;
1100 }
1101
1102 struct format_field *perf_evsel__field(struct perf_evsel *evsel, const char *name)
1103 {
1104         return pevent_find_field(evsel->tp_format, name);
1105 }
1106
1107 void *perf_evsel__rawptr(struct perf_evsel *evsel, struct perf_sample *sample,
1108                          const char *name)
1109 {
1110         struct format_field *field = perf_evsel__field(evsel, name);
1111         int offset;
1112
1113         if (!field)
1114                 return NULL;
1115
1116         offset = field->offset;
1117
1118         if (field->flags & FIELD_IS_DYNAMIC) {
1119                 offset = *(int *)(sample->raw_data + field->offset);
1120                 offset &= 0xffff;
1121         }
1122
1123         return sample->raw_data + offset;
1124 }
1125
1126 u64 perf_evsel__intval(struct perf_evsel *evsel, struct perf_sample *sample,
1127                        const char *name)
1128 {
1129         struct format_field *field = perf_evsel__field(evsel, name);
1130         void *ptr;
1131         u64 value;
1132
1133         if (!field)
1134                 return 0;
1135
1136         ptr = sample->raw_data + field->offset;
1137
1138         switch (field->size) {
1139         case 1:
1140                 return *(u8 *)ptr;
1141         case 2:
1142                 value = *(u16 *)ptr;
1143                 break;
1144         case 4:
1145                 value = *(u32 *)ptr;
1146                 break;
1147         case 8:
1148                 value = *(u64 *)ptr;
1149                 break;
1150         default:
1151                 return 0;
1152         }
1153
1154         if (!evsel->needs_swap)
1155                 return value;
1156
1157         switch (field->size) {
1158         case 2:
1159                 return bswap_16(value);
1160         case 4:
1161                 return bswap_32(value);
1162         case 8:
1163                 return bswap_64(value);
1164         default:
1165                 return 0;
1166         }
1167
1168         return 0;
1169 }