arch/s390/oprofile/hwsampler.c

   1 /**
   2  * arch/s390/oprofile/hwsampler.c
   3  *
   4  * Copyright IBM Corp. 2010
   5  * Author: Heinz Graalfs <graalfs@de.ibm.com>
   6  */
   7
   8 #include <linux/kernel_stat.h>
   9 #include <linux/kernel.h>
  10 #include <linux/module.h>
  11 #include <linux/smp.h>
  12 #include <linux/errno.h>
  13 #include <linux/workqueue.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/notifier.h>
  16 #include <linux/cpu.h>
  17 #include <linux/semaphore.h>
  18 #include <linux/oom.h>
  19 #include <linux/oprofile.h>
  20
  21 #include <asm/lowcore.h>
  22 #include <asm/s390_ext.h>
  23
  24 #include "hwsampler.h"
  25
  26 #define MAX_NUM_SDB 511
  27 #define MIN_NUM_SDB 1
  28
  29 #define ALERT_REQ_MASK   0x4000000000000000ul
  30 #define BUFFER_FULL_MASK 0x8000000000000000ul
  31
  32 #define EI_IEA      (1 << 31)   /* invalid entry address              */
  33 #define EI_ISE      (1 << 30)   /* incorrect SDBT entry               */
  34 #define EI_PRA      (1 << 29)   /* program request alert              */
  35 #define EI_SACA     (1 << 23)   /* sampler authorization change alert */
  36 #define EI_LSDA     (1 << 22)   /* loss of sample data alert          */
  37
  38 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
  39
  40 struct hws_execute_parms {
  41         void *buffer;
  42         signed int rc;
  43 };
  44
  45 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
  46 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
  47
  48 static DEFINE_MUTEX(hws_sem);
  49 static DEFINE_MUTEX(hws_sem_oom);
  50
  51 static unsigned char hws_flush_all;
  52 static unsigned int hws_oom;
  53 static struct workqueue_struct *hws_wq;
  54
  55 static unsigned int hws_state;
  56 enum {
  57         HWS_INIT = 1,
  58         HWS_DEALLOCATED,
  59         HWS_STOPPED,
  60         HWS_STARTED,
  61         HWS_STOPPING };
  62
  63 /* set to 1 if called by kernel during memory allocation */
  64 static unsigned char oom_killer_was_active;
  65 /* size of SDBT and SDB as of allocate API */
  66 static unsigned long num_sdbt = 100;
  67 static unsigned long num_sdb = 511;
  68 /* sampling interval (machine cycles) */
  69 static unsigned long interval;
  70
  71 static unsigned long min_sampler_rate;
  72 static unsigned long max_sampler_rate;
  73
  74 static int ssctl(void *buffer)
  75 {
  76         int cc;
  77
  78         /* set in order to detect a program check */
  79         cc = 1;
  80
  81         asm volatile(
  82                 "0: .insn s,0xB2870000,0(%1)\n"
  83                 "1: ipm %0\n"
  84                 "   srl %0,28\n"
  85                 "2:\n"
  86                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
  87                 : "+d" (cc), "+a" (buffer)
  88                 : "m" (*((struct hws_ssctl_request_block *)buffer))
  89                 : "cc", "memory");
  90
  91         return cc ? -EINVAL : 0 ;
  92 }
  93
  94 static int qsi(void *buffer)
  95 {
  96         int cc;
  97         cc = 1;
  98
  99         asm volatile(
 100                 "0: .insn s,0xB2860000,0(%1)\n"
 101                 "1: lhi %0,0\n"
 102                 "2:\n"
 103                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
 104                 : "=d" (cc), "+a" (buffer)
 105                 : "m" (*((struct hws_qsi_info_block *)buffer))
 106                 : "cc", "memory");
 107
 108         return cc ? -EINVAL : 0;
 109 }
 110
 111 static void execute_qsi(void *parms)
 112 {
 113         struct hws_execute_parms *ep = parms;
 114
 115         ep->rc = qsi(ep->buffer);
 116 }
 117
 118 static void execute_ssctl(void *parms)
 119 {
 120         struct hws_execute_parms *ep = parms;
 121
 122         ep->rc = ssctl(ep->buffer);
 123 }
 124
 125 static int smp_ctl_ssctl_stop(int cpu)
 126 {
 127         int rc;
 128         struct hws_execute_parms ep;
 129         struct hws_cpu_buffer *cb;
 130
 131         cb = &per_cpu(sampler_cpu_buffer, cpu);
 132
 133         cb->ssctl.es = 0;
 134         cb->ssctl.cs = 0;
 135
 136         ep.buffer = &cb->ssctl;
 137         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 138         rc = ep.rc;
 139         if (rc) {
 140                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 141                 dump_stack();
 142         }
 143
 144         ep.buffer = &cb->qsi;
 145         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 146
 147         if (cb->qsi.es || cb->qsi.cs) {
 148                 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
 149                 dump_stack();
 150         }
 151
 152         return rc;
 153 }
 154
 155 static int smp_ctl_ssctl_deactivate(int cpu)
 156 {
 157         int rc;
 158         struct hws_execute_parms ep;
 159         struct hws_cpu_buffer *cb;
 160
 161         cb = &per_cpu(sampler_cpu_buffer, cpu);
 162
 163         cb->ssctl.es = 1;
 164         cb->ssctl.cs = 0;
 165
 166         ep.buffer = &cb->ssctl;
 167         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 168         rc = ep.rc;
 169         if (rc)
 170                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 171
 172         ep.buffer = &cb->qsi;
 173         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 174
 175         if (cb->qsi.cs)
 176                 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
 177
 178         return rc;
 179 }
 180
 181 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
 182 {
 183         int rc;
 184         struct hws_execute_parms ep;
 185         struct hws_cpu_buffer *cb;
 186
 187         cb = &per_cpu(sampler_cpu_buffer, cpu);
 188
 189         cb->ssctl.h = 1;
 190         cb->ssctl.tear = cb->first_sdbt;
 191         cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
 192         cb->ssctl.interval = interval;
 193         cb->ssctl.es = 1;
 194         cb->ssctl.cs = 1;
 195
 196         ep.buffer = &cb->ssctl;
 197         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 198         rc = ep.rc;
 199         if (rc)
 200                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 201
 202         ep.buffer = &cb->qsi;
 203         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 204         if (ep.rc)
 205                 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
 206
 207         return rc;
 208 }
 209
 210 static int smp_ctl_qsi(int cpu)
 211 {
 212         struct hws_execute_parms ep;
 213         struct hws_cpu_buffer *cb;
 214
 215         cb = &per_cpu(sampler_cpu_buffer, cpu);
 216
 217         ep.buffer = &cb->qsi;
 218         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 219
 220         return ep.rc;
 221 }
 222
 223 static inline unsigned long *trailer_entry_ptr(unsigned long v)
 224 {
 225         void *ret;
 226
 227         ret = (void *)v;
 228         ret += PAGE_SIZE;
 229         ret -= sizeof(struct hws_trailer_entry);
 230
 231         return (unsigned long *) ret;
 232 }
 233
 234 /* prototypes for external interrupt handler and worker */
 235 static void hws_ext_handler(unsigned int ext_int_code,
 236                                 unsigned int param32, unsigned long param64);
 237
 238 static void worker(struct work_struct *work);
 239
 240 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
 241                                 unsigned long *dear);
 242
 243 static void init_all_cpu_buffers(void)
 244 {
 245         int cpu;
 246         struct hws_cpu_buffer *cb;
 247
 248         for_each_online_cpu(cpu) {
 249                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 250                 memset(cb, 0, sizeof(struct hws_cpu_buffer));
 251         }
 252 }
 253
 254 static int is_link_entry(unsigned long *s)
 255 {
 256         return *s & 0x1ul ? 1 : 0;
 257 }
 258
 259 static unsigned long *get_next_sdbt(unsigned long *s)
 260 {
 261         return (unsigned long *) (*s & ~0x1ul);
 262 }
 263
 264 static int prepare_cpu_buffers(void)
 265 {
 266         int cpu;
 267         int rc;
 268         struct hws_cpu_buffer *cb;
 269
 270         rc = 0;
 271         for_each_online_cpu(cpu) {
 272                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 273                 atomic_set(&cb->ext_params, 0);
 274                 cb->worker_entry = 0;
 275                 cb->sample_overflow = 0;
 276                 cb->req_alert = 0;
 277                 cb->incorrect_sdbt_entry = 0;
 278                 cb->invalid_entry_address = 0;
 279                 cb->loss_of_sample_data = 0;
 280                 cb->sample_auth_change_alert = 0;
 281                 cb->finish = 0;
 282                 cb->oom = 0;
 283                 cb->stop_mode = 0;
 284         }
 285
 286         return rc;
 287 }
 288
 289 /*
 290  * allocate_sdbt() - allocate sampler memory
 291  * @cpu: the cpu for which sampler memory is allocated
 292  *
 293  * A 4K page is allocated for each requested SDBT.
 294  * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
 295  * Set ALERT_REQ mask in each SDBs trailer.
 296  * Returns zero if successful, <0 otherwise.
 297  */
 298 static int allocate_sdbt(int cpu)
 299 {
 300         int j, k, rc;
 301         unsigned long *sdbt;
 302         unsigned long  sdb;
 303         unsigned long *tail;
 304         unsigned long *trailer;
 305         struct hws_cpu_buffer *cb;
 306
 307         cb = &per_cpu(sampler_cpu_buffer, cpu);
 308
 309         if (cb->first_sdbt)
 310                 return -EINVAL;
 311
 312         sdbt = NULL;
 313         tail = sdbt;
 314
 315         for (j = 0; j < num_sdbt; j++) {
 316                 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 317
 318                 mutex_lock(&hws_sem_oom);
 319                 /* OOM killer might have been activated */
 320                 barrier();
 321                 if (oom_killer_was_active || !sdbt) {
 322                         if (sdbt)
 323                                 free_page((unsigned long)sdbt);
 324
 325                         goto allocate_sdbt_error;
 326                 }
 327                 if (cb->first_sdbt == 0)
 328                         cb->first_sdbt = (unsigned long)sdbt;
 329
 330                 /* link current page to tail of chain */
 331                 if (tail)
 332                         *tail = (unsigned long)(void *)sdbt + 1;
 333
 334                 mutex_unlock(&hws_sem_oom);
 335
 336                 for (k = 0; k < num_sdb; k++) {
 337                         /* get and set SDB page */
 338                         sdb = get_zeroed_page(GFP_KERNEL);
 339
 340                         mutex_lock(&hws_sem_oom);
 341                         /* OOM killer might have been activated */
 342                         barrier();
 343                         if (oom_killer_was_active || !sdb) {
 344                                 if (sdb)
 345                                         free_page(sdb);
 346
 347                                 goto allocate_sdbt_error;
 348                         }
 349                         *sdbt = sdb;
 350                         trailer = trailer_entry_ptr(*sdbt);
 351                         *trailer = ALERT_REQ_MASK;
 352                         sdbt++;
 353                         mutex_unlock(&hws_sem_oom);
 354                 }
 355                 tail = sdbt;
 356         }
 357         mutex_lock(&hws_sem_oom);
 358         if (oom_killer_was_active)
 359                 goto allocate_sdbt_error;
 360
 361         rc = 0;
 362         if (tail)
 363                 *tail = (unsigned long)
 364                         ((void *)cb->first_sdbt) + 1;
 365
 366 allocate_sdbt_exit:
 367         mutex_unlock(&hws_sem_oom);
 368         return rc;
 369
 370 allocate_sdbt_error:
 371         rc = -ENOMEM;
 372         goto allocate_sdbt_exit;
 373 }
 374
 375 /*
 376  * deallocate_sdbt() - deallocate all sampler memory
 377  *
 378  * For each online CPU all SDBT trees are deallocated.
 379  * Returns the number of freed pages.
 380  */
 381 static int deallocate_sdbt(void)
 382 {
 383         int cpu;
 384         int counter;
 385
 386         counter = 0;
 387
 388         for_each_online_cpu(cpu) {
 389                 unsigned long start;
 390                 unsigned long sdbt;
 391                 unsigned long *curr;
 392                 struct hws_cpu_buffer *cb;
 393
 394                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 395
 396                 if (!cb->first_sdbt)
 397                         continue;
 398
 399                 sdbt = cb->first_sdbt;
 400                 curr = (unsigned long *) sdbt;
 401                 start = sdbt;
 402
 403                 /* we'll free the SDBT after all SDBs are processed... */
 404                 while (1) {
 405                         if (!*curr || !sdbt)
 406                                 break;
 407
 408                         /* watch for link entry reset if found */
 409                         if (is_link_entry(curr)) {
 410                                 curr = get_next_sdbt(curr);
 411                                 if (sdbt)
 412                                         free_page(sdbt);
 413
 414                                 /* we are done if we reach the start */
 415                                 if ((unsigned long) curr == start)
 416                                         break;
 417                                 else
 418                                         sdbt = (unsigned long) curr;
 419                         } else {
 420                                 /* process SDB pointer */
 421                                 if (*curr) {
 422                                         free_page(*curr);
 423                                         curr++;
 424                                 }
 425                         }
 426                         counter++;
 427                 }
 428                 cb->first_sdbt = 0;
 429         }
 430         return counter;
 431 }
 432
 433 static int start_sampling(int cpu)
 434 {
 435         int rc;
 436         struct hws_cpu_buffer *cb;
 437
 438         cb = &per_cpu(sampler_cpu_buffer, cpu);
 439         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
 440         if (rc) {
 441                 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
 442                 goto start_exit;
 443         }
 444
 445         rc = -EINVAL;
 446         if (!cb->qsi.es) {
 447                 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
 448                 goto start_exit;
 449         }
 450
 451         if (!cb->qsi.cs) {
 452                 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
 453                 goto start_exit;
 454         }
 455
 456         printk(KERN_INFO
 457                 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
 458                 cpu, interval);
 459
 460         rc = 0;
 461
 462 start_exit:
 463         return rc;
 464 }
 465
 466 static int stop_sampling(int cpu)
 467 {
 468         unsigned long v;
 469         int rc;
 470         struct hws_cpu_buffer *cb;
 471
 472         rc = smp_ctl_qsi(cpu);
 473         WARN_ON(rc);
 474
 475         cb = &per_cpu(sampler_cpu_buffer, cpu);
 476         if (!rc && !cb->qsi.es)
 477                 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
 478
 479         rc = smp_ctl_ssctl_stop(cpu);
 480         if (rc) {
 481                 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
 482                                 cpu, rc);
 483                 goto stop_exit;
 484         }
 485
 486         printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
 487
 488 stop_exit:
 489         v = cb->req_alert;
 490         if (v)
 491                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
 492                                 " count=%lu.\n", cpu, v);
 493
 494         v = cb->loss_of_sample_data;
 495         if (v)
 496                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
 497                                 " count=%lu.\n", cpu, v);
 498
 499         v = cb->invalid_entry_address;
 500         if (v)
 501                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
 502                                 " count=%lu.\n", cpu, v);
 503
 504         v = cb->incorrect_sdbt_entry;
 505         if (v)
 506                 printk(KERN_ERR
 507                                 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
 508                                 " count=%lu.\n", cpu, v);
 509
 510         v = cb->sample_auth_change_alert;
 511         if (v)
 512                 printk(KERN_ERR
 513                                 "hwsampler: CPU %d CPUMF Sample authorization change,"
 514                                 " count=%lu.\n", cpu, v);
 515
 516         return rc;
 517 }
 518
 519 static int check_hardware_prerequisites(void)
 520 {
 521         if (!test_facility(68))
 522                 return -EOPNOTSUPP;
 523         return 0;
 524 }
 525 /*
 526  * hws_oom_callback() - the OOM callback function
 527  *
 528  * In case the callback is invoked during memory allocation for the
 529  *  hw sampler, all obtained memory is deallocated and a flag is set
 530  *  so main sampler memory allocation can exit with a failure code.
 531  * In case the callback is invoked during sampling the hw sampler
 532  *  is deactivated for all CPUs.
 533  */
 534 static int hws_oom_callback(struct notifier_block *nfb,
 535         unsigned long dummy, void *parm)
 536 {
 537         unsigned long *freed;
 538         int cpu;
 539         struct hws_cpu_buffer *cb;
 540
 541         freed = parm;
 542
 543         mutex_lock(&hws_sem_oom);
 544
 545         if (hws_state == HWS_DEALLOCATED) {
 546                 /* during memory allocation */
 547                 if (oom_killer_was_active == 0) {
 548                         oom_killer_was_active = 1;
 549                         *freed += deallocate_sdbt();
 550                 }
 551         } else {
 552                 int i;
 553                 cpu = get_cpu();
 554                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 555
 556                 if (!cb->oom) {
 557                         for_each_online_cpu(i) {
 558                                 smp_ctl_ssctl_deactivate(i);
 559                                 cb->oom = 1;
 560                         }
 561                         cb->finish = 1;
 562
 563                         printk(KERN_INFO
 564                                 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
 565                                 cpu);
 566                 }
 567         }
 568
 569         mutex_unlock(&hws_sem_oom);
 570
 571         return NOTIFY_OK;
 572 }
 573
 574 static struct notifier_block hws_oom_notifier = {
 575         .notifier_call = hws_oom_callback
 576 };
 577
 578 static int hws_cpu_callback(struct notifier_block *nfb,
 579         unsigned long action, void *hcpu)
 580 {
 581         /* We do not have sampler space available for all possible CPUs.
 582            All CPUs should be online when hw sampling is activated. */
 583         return NOTIFY_BAD;
 584 }
 585
 586 static struct notifier_block hws_cpu_notifier = {
 587         .notifier_call = hws_cpu_callback
 588 };
 589
 590 /**
 591  * hwsampler_deactivate() - set hardware sampling temporarily inactive
 592  * @cpu:  specifies the CPU to be set inactive.
 593  *
 594  * Returns 0 on success, !0 on failure.
 595  */
 596 int hwsampler_deactivate(unsigned int cpu)
 597 {
 598         /*
 599          * Deactivate hw sampling temporarily and flush the buffer
 600          * by pushing all the pending samples to oprofile buffer.
 601          *
 602          * This function can be called under one of the following conditions:
 603          *     Memory unmap, task is exiting.
 604          */
 605         int rc;
 606         struct hws_cpu_buffer *cb;
 607
 608         rc = 0;
 609         mutex_lock(&hws_sem);
 610
 611         cb = &per_cpu(sampler_cpu_buffer, cpu);
 612         if (hws_state == HWS_STARTED) {
 613                 rc = smp_ctl_qsi(cpu);
 614                 WARN_ON(rc);
 615                 if (cb->qsi.cs) {
 616                         rc = smp_ctl_ssctl_deactivate(cpu);
 617                         if (rc) {
 618                                 printk(KERN_INFO
 619                                 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
 620                                 cb->finish = 1;
 621                                 hws_state = HWS_STOPPING;
 622                         } else  {
 623                                 hws_flush_all = 1;
 624                                 /* Add work to queue to read pending samples.*/
 625                                 queue_work_on(cpu, hws_wq, &cb->worker);
 626                         }
 627                 }
 628         }
 629         mutex_unlock(&hws_sem);
 630
 631         if (hws_wq)
 632                 flush_workqueue(hws_wq);
 633
 634         return rc;
 635 }
 636
 637 /**
 638  * hwsampler_activate() - activate/resume hardware sampling which was deactivated
 639  * @cpu:  specifies the CPU to be set active.
 640  *
 641  * Returns 0 on success, !0 on failure.
 642  */
 643 int hwsampler_activate(unsigned int cpu)
 644 {
 645         /*
 646          * Re-activate hw sampling. This should be called in pair with
 647          * hwsampler_deactivate().
 648          */
 649         int rc;
 650         struct hws_cpu_buffer *cb;
 651
 652         rc = 0;
 653         mutex_lock(&hws_sem);
 654
 655         cb = &per_cpu(sampler_cpu_buffer, cpu);
 656         if (hws_state == HWS_STARTED) {
 657                 rc = smp_ctl_qsi(cpu);
 658                 WARN_ON(rc);
 659                 if (!cb->qsi.cs) {
 660                         hws_flush_all = 0;
 661                         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
 662                         if (rc) {
 663                                 printk(KERN_ERR
 664                                 "CPU %d, CPUMF activate sampling failed.\n",
 665                                          cpu);
 666                         }
 667                 }
 668         }
 669
 670         mutex_unlock(&hws_sem);
 671
 672         return rc;
 673 }
 674
 675 static void hws_ext_handler(unsigned int ext_int_code,
 676                             unsigned int param32, unsigned long param64)
 677 {
 678         struct hws_cpu_buffer *cb;
 679
 680         kstat_cpu(smp_processor_id()).irqs[EXTINT_CPM]++;
 681         cb = &__get_cpu_var(sampler_cpu_buffer);
 682         atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
 683         if (hws_wq)
 684                 queue_work(hws_wq, &cb->worker);
 685 }
 686
 687 static int check_qsi_on_setup(void)
 688 {
 689         int rc;
 690         unsigned int cpu;
 691         struct hws_cpu_buffer *cb;
 692
 693         for_each_online_cpu(cpu) {
 694                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 695                 rc = smp_ctl_qsi(cpu);
 696                 WARN_ON(rc);
 697                 if (rc)
 698                         return -EOPNOTSUPP;
 699
 700                 if (!cb->qsi.as) {
 701                         printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
 702                         return -EINVAL;
 703                 }
 704
 705                 if (cb->qsi.es) {
 706                         printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
 707                         rc = smp_ctl_ssctl_stop(cpu);
 708                         if (rc)
 709                                 return -EINVAL;
 710
 711                         printk(KERN_INFO
 712                                 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
 713                 }
 714         }
 715         return 0;
 716 }
 717
 718 static int check_qsi_on_start(void)
 719 {
 720         unsigned int cpu;
 721         int rc;
 722         struct hws_cpu_buffer *cb;
 723
 724         for_each_online_cpu(cpu) {
 725                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 726                 rc = smp_ctl_qsi(cpu);
 727                 WARN_ON(rc);
 728
 729                 if (!cb->qsi.as)
 730                         return -EINVAL;
 731
 732                 if (cb->qsi.es)
 733                         return -EINVAL;
 734
 735                 if (cb->qsi.cs)
 736                         return -EINVAL;
 737         }
 738         return 0;
 739 }
 740
 741 static void worker_on_start(unsigned int cpu)
 742 {
 743         struct hws_cpu_buffer *cb;
 744
 745         cb = &per_cpu(sampler_cpu_buffer, cpu);
 746         cb->worker_entry = cb->first_sdbt;
 747 }
 748
 749 static int worker_check_error(unsigned int cpu, int ext_params)
 750 {
 751         int rc;
 752         unsigned long *sdbt;
 753         struct hws_cpu_buffer *cb;
 754
 755         rc = 0;
 756         cb = &per_cpu(sampler_cpu_buffer, cpu);
 757         sdbt = (unsigned long *) cb->worker_entry;
 758
 759         if (!sdbt || !*sdbt)
 760                 return -EINVAL;
 761
 762         if (ext_params & EI_PRA)
 763                 cb->req_alert++;
 764
 765         if (ext_params & EI_LSDA)
 766                 cb->loss_of_sample_data++;
 767
 768         if (ext_params & EI_IEA) {
 769                 cb->invalid_entry_address++;
 770                 rc = -EINVAL;
 771         }
 772
 773         if (ext_params & EI_ISE) {
 774                 cb->incorrect_sdbt_entry++;
 775                 rc = -EINVAL;
 776         }
 777
 778         if (ext_params & EI_SACA) {
 779                 cb->sample_auth_change_alert++;
 780                 rc = -EINVAL;
 781         }
 782
 783         return rc;
 784 }
 785
 786 static void worker_on_finish(unsigned int cpu)
 787 {
 788         int rc, i;
 789         struct hws_cpu_buffer *cb;
 790
 791         cb = &per_cpu(sampler_cpu_buffer, cpu);
 792
 793         if (cb->finish) {
 794                 rc = smp_ctl_qsi(cpu);
 795                 WARN_ON(rc);
 796                 if (cb->qsi.es) {
 797                         printk(KERN_INFO
 798                                 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
 799                                 cpu);
 800                         rc = smp_ctl_ssctl_stop(cpu);
 801                         if (rc)
 802                                 printk(KERN_INFO
 803                                         "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
 804                                         cpu);
 805
 806                         for_each_online_cpu(i) {
 807                                 if (i == cpu)
 808                                         continue;
 809                                 if (!cb->finish) {
 810                                         cb->finish = 1;
 811                                         queue_work_on(i, hws_wq,
 812                                                 &cb->worker);
 813                                 }
 814                         }
 815                 }
 816         }
 817 }
 818
 819 static void worker_on_interrupt(unsigned int cpu)
 820 {
 821         unsigned long *sdbt;
 822         unsigned char done;
 823         struct hws_cpu_buffer *cb;
 824
 825         cb = &per_cpu(sampler_cpu_buffer, cpu);
 826
 827         sdbt = (unsigned long *) cb->worker_entry;
 828
 829         done = 0;
 830         /* do not proceed if stop was entered,
 831          * forget the buffers not yet processed */
 832         while (!done && !cb->stop_mode) {
 833                 unsigned long *trailer;
 834                 struct hws_trailer_entry *te;
 835                 unsigned long *dear = 0;
 836
 837                 trailer = trailer_entry_ptr(*sdbt);
 838                 /* leave loop if no more work to do */
 839                 if (!(*trailer & BUFFER_FULL_MASK)) {
 840                         done = 1;
 841                         if (!hws_flush_all)
 842                                 continue;
 843                 }
 844
 845                 te = (struct hws_trailer_entry *)trailer;
 846                 cb->sample_overflow += te->overflow;
 847
 848                 add_samples_to_oprofile(cpu, sdbt, dear);
 849
 850                 /* reset trailer */
 851                 xchg((unsigned char *) te, 0x40);
 852
 853                 /* advance to next sdb slot in current sdbt */
 854                 sdbt++;
 855                 /* in case link bit is set use address w/o link bit */
 856                 if (is_link_entry(sdbt))
 857                         sdbt = get_next_sdbt(sdbt);
 858
 859                 cb->worker_entry = (unsigned long)sdbt;
 860         }
 861 }
 862
 863 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
 864                 unsigned long *dear)
 865 {
 866         struct hws_data_entry *sample_data_ptr;
 867         unsigned long *trailer;
 868
 869         trailer = trailer_entry_ptr(*sdbt);
 870         if (dear) {
 871                 if (dear > trailer)
 872                         return;
 873                 trailer = dear;
 874         }
 875
 876         sample_data_ptr = (struct hws_data_entry *)(*sdbt);
 877
 878         while ((unsigned long *)sample_data_ptr < trailer) {
 879                 struct pt_regs *regs = NULL;
 880                 struct task_struct *tsk = NULL;
 881
 882                 /*
 883                  * Check sampling mode, 1 indicates basic (=customer) sampling
 884                  * mode.
 885                  */
 886                 if (sample_data_ptr->def != 1) {
 887                         /* sample slot is not yet written */
 888                         break;
 889                 } else {
 890                         /* make sure we don't use it twice,
 891                          * the next time the sampler will set it again */
 892                         sample_data_ptr->def = 0;
 893                 }
 894
 895                 /* Get pt_regs. */
 896                 if (sample_data_ptr->P == 1) {
 897                         /* userspace sample */
 898                         unsigned int pid = sample_data_ptr->prim_asn;
 899                         rcu_read_lock();
 900                         tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
 901                         if (tsk)
 902                                 regs = task_pt_regs(tsk);
 903                         rcu_read_unlock();
 904                 } else {
 905                         /* kernelspace sample */
 906                         regs = task_pt_regs(current);
 907                 }
 908
 909                 mutex_lock(&hws_sem);
 910                 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
 911                                 !sample_data_ptr->P, tsk);
 912                 mutex_unlock(&hws_sem);
 913
 914                 sample_data_ptr++;
 915         }
 916 }
 917
 918 static void worker(struct work_struct *work)
 919 {
 920         unsigned int cpu;
 921         int ext_params;
 922         struct hws_cpu_buffer *cb;
 923
 924         cb = container_of(work, struct hws_cpu_buffer, worker);
 925         cpu = smp_processor_id();
 926         ext_params = atomic_xchg(&cb->ext_params, 0);
 927
 928         if (!cb->worker_entry)
 929                 worker_on_start(cpu);
 930
 931         if (worker_check_error(cpu, ext_params))
 932                 return;
 933
 934         if (!cb->finish)
 935                 worker_on_interrupt(cpu);
 936
 937         if (cb->finish)
 938                 worker_on_finish(cpu);
 939 }
 940
 941 /**
 942  * hwsampler_allocate() - allocate memory for the hardware sampler
 943  * @sdbt:  number of SDBTs per online CPU (must be > 0)
 944  * @sdb:   number of SDBs per SDBT (minimum 1, maximum 511)
 945  *
 946  * Returns 0 on success, !0 on failure.
 947  */
 948 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
 949 {
 950         int cpu, rc;
 951         mutex_lock(&hws_sem);
 952
 953         rc = -EINVAL;
 954         if (hws_state != HWS_DEALLOCATED)
 955                 goto allocate_exit;
 956
 957         if (sdbt < 1)
 958                 goto allocate_exit;
 959
 960         if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
 961                 goto allocate_exit;
 962
 963         num_sdbt = sdbt;
 964         num_sdb = sdb;
 965
 966         oom_killer_was_active = 0;
 967         register_oom_notifier(&hws_oom_notifier);
 968
 969         for_each_online_cpu(cpu) {
 970                 if (allocate_sdbt(cpu)) {
 971                         unregister_oom_notifier(&hws_oom_notifier);
 972                         goto allocate_error;
 973                 }
 974         }
 975         unregister_oom_notifier(&hws_oom_notifier);
 976         if (oom_killer_was_active)
 977                 goto allocate_error;
 978
 979         hws_state = HWS_STOPPED;
 980         rc = 0;
 981
 982 allocate_exit:
 983         mutex_unlock(&hws_sem);
 984         return rc;
 985
 986 allocate_error:
 987         rc = -ENOMEM;
 988         printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
 989         goto allocate_exit;
 990 }
 991
 992 /**
 993  * hwsampler_deallocate() - deallocate hardware sampler memory
 994  *
 995  * Returns 0 on success, !0 on failure.
 996  */
 997 int hwsampler_deallocate()
 998 {
 999         int rc;
1000
1001         mutex_lock(&hws_sem);
1002
1003         rc = -EINVAL;
1004         if (hws_state != HWS_STOPPED)
1005                 goto deallocate_exit;
1006
1007         ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1008         deallocate_sdbt();
1009
1010         hws_state = HWS_DEALLOCATED;
1011         rc = 0;
1012
1013 deallocate_exit:
1014         mutex_unlock(&hws_sem);
1015
1016         return rc;
1017 }
1018
1019 unsigned long hwsampler_query_min_interval(void)
1020 {
1021         return min_sampler_rate;
1022 }
1023
1024 unsigned long hwsampler_query_max_interval(void)
1025 {
1026         return max_sampler_rate;
1027 }
1028
1029 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1030 {
1031         struct hws_cpu_buffer *cb;
1032
1033         cb = &per_cpu(sampler_cpu_buffer, cpu);
1034
1035         return cb->sample_overflow;
1036 }
1037
1038 int hwsampler_setup()
1039 {
1040         int rc;
1041         int cpu;
1042         struct hws_cpu_buffer *cb;
1043
1044         mutex_lock(&hws_sem);
1045
1046         rc = -EINVAL;
1047         if (hws_state)
1048                 goto setup_exit;
1049
1050         hws_state = HWS_INIT;
1051
1052         init_all_cpu_buffers();
1053
1054         rc = check_hardware_prerequisites();
1055         if (rc)
1056                 goto setup_exit;
1057
1058         rc = check_qsi_on_setup();
1059         if (rc)
1060                 goto setup_exit;
1061
1062         rc = -EINVAL;
1063         hws_wq = create_workqueue("hwsampler");
1064         if (!hws_wq)
1065                 goto setup_exit;
1066
1067         register_cpu_notifier(&hws_cpu_notifier);
1068
1069         for_each_online_cpu(cpu) {
1070                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1071                 INIT_WORK(&cb->worker, worker);
1072                 rc = smp_ctl_qsi(cpu);
1073                 WARN_ON(rc);
1074                 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1075                         if (min_sampler_rate) {
1076                                 printk(KERN_WARNING
1077                                         "hwsampler: different min sampler rate values.\n");
1078                                 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1079                                         min_sampler_rate =
1080                                                 cb->qsi.min_sampl_rate;
1081                         } else
1082                                 min_sampler_rate = cb->qsi.min_sampl_rate;
1083                 }
1084                 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1085                         if (max_sampler_rate) {
1086                                 printk(KERN_WARNING
1087                                         "hwsampler: different max sampler rate values.\n");
1088                                 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1089                                         max_sampler_rate =
1090                                                 cb->qsi.max_sampl_rate;
1091                         } else
1092                                 max_sampler_rate = cb->qsi.max_sampl_rate;
1093                 }
1094         }
1095         register_external_interrupt(0x1407, hws_ext_handler);
1096
1097         hws_state = HWS_DEALLOCATED;
1098         rc = 0;
1099
1100 setup_exit:
1101         mutex_unlock(&hws_sem);
1102         return rc;
1103 }
1104
1105 int hwsampler_shutdown()
1106 {
1107         int rc;
1108
1109         mutex_lock(&hws_sem);
1110
1111         rc = -EINVAL;
1112         if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1113                 mutex_unlock(&hws_sem);
1114
1115                 if (hws_wq)
1116                         flush_workqueue(hws_wq);
1117
1118                 mutex_lock(&hws_sem);
1119
1120                 if (hws_state == HWS_STOPPED) {
1121                         ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1122                         deallocate_sdbt();
1123                 }
1124                 if (hws_wq) {
1125                         destroy_workqueue(hws_wq);
1126                         hws_wq = NULL;
1127                 }
1128
1129                 unregister_external_interrupt(0x1407, hws_ext_handler);
1130                 hws_state = HWS_INIT;
1131                 rc = 0;
1132         }
1133         mutex_unlock(&hws_sem);
1134
1135         unregister_cpu_notifier(&hws_cpu_notifier);
1136
1137         return rc;
1138 }
1139
1140 /**
1141  * hwsampler_start_all() - start hardware sampling on all online CPUs
1142  * @rate:  specifies the used interval when samples are taken
1143  *
1144  * Returns 0 on success, !0 on failure.
1145  */
1146 int hwsampler_start_all(unsigned long rate)
1147 {
1148         int rc, cpu;
1149
1150         mutex_lock(&hws_sem);
1151
1152         hws_oom = 0;
1153
1154         rc = -EINVAL;
1155         if (hws_state != HWS_STOPPED)
1156                 goto start_all_exit;
1157
1158         interval = rate;
1159
1160         /* fail if rate is not valid */
1161         if (interval < min_sampler_rate || interval > max_sampler_rate)
1162                 goto start_all_exit;
1163
1164         rc = check_qsi_on_start();
1165         if (rc)
1166                 goto start_all_exit;
1167
1168         rc = prepare_cpu_buffers();
1169         if (rc)
1170                 goto start_all_exit;
1171
1172         for_each_online_cpu(cpu) {
1173                 rc = start_sampling(cpu);
1174                 if (rc)
1175                         break;
1176         }
1177         if (rc) {
1178                 for_each_online_cpu(cpu) {
1179                         stop_sampling(cpu);
1180                 }
1181                 goto start_all_exit;
1182         }
1183         hws_state = HWS_STARTED;
1184         rc = 0;
1185
1186 start_all_exit:
1187         mutex_unlock(&hws_sem);
1188
1189         if (rc)
1190                 return rc;
1191
1192         register_oom_notifier(&hws_oom_notifier);
1193         hws_oom = 1;
1194         hws_flush_all = 0;
1195         /* now let them in, 1407 CPUMF external interrupts */
1196         ctl_set_bit(0, 5); /* set CR0 bit 58 */
1197
1198         return 0;
1199 }
1200
1201 /**
1202  * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1203  *
1204  * Returns 0 on success, !0 on failure.
1205  */
1206 int hwsampler_stop_all()
1207 {
1208         int tmp_rc, rc, cpu;
1209         struct hws_cpu_buffer *cb;
1210
1211         mutex_lock(&hws_sem);
1212
1213         rc = 0;
1214         if (hws_state == HWS_INIT) {
1215                 mutex_unlock(&hws_sem);
1216                 return rc;
1217         }
1218         hws_state = HWS_STOPPING;
1219         mutex_unlock(&hws_sem);
1220
1221         for_each_online_cpu(cpu) {
1222                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1223                 cb->stop_mode = 1;
1224                 tmp_rc = stop_sampling(cpu);
1225                 if (tmp_rc)
1226                         rc = tmp_rc;
1227         }
1228
1229         if (hws_wq)
1230                 flush_workqueue(hws_wq);
1231
1232         mutex_lock(&hws_sem);
1233         if (hws_oom) {
1234                 unregister_oom_notifier(&hws_oom_notifier);
1235                 hws_oom = 0;
1236         }
1237         hws_state = HWS_STOPPED;
1238         mutex_unlock(&hws_sem);
1239
1240         return rc;
1241 }