arch/powerpc/oprofile/cell/spu_task_sync.c

   1 /*
   2  * Cell Broadband Engine OProfile Support
   3  *
   4  * (C) Copyright IBM Corporation 2006
   5  *
   6  * Author: Maynard Johnson <maynardj@us.ibm.com>
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License
  10  * as published by the Free Software Foundation; either version
  11  * 2 of the License, or (at your option) any later version.
  12  */
  13
  14 /* The purpose of this file is to handle SPU event task switching
  15  * and to record SPU context information into the OProfile
  16  * event buffer.
  17  *
  18  * Additionally, the spu_sync_buffer function is provided as a helper
  19  * for recoding actual SPU program counter samples to the event buffer.
  20  */
  21 #include <linux/dcookies.h>
  22 #include <linux/kref.h>
  23 #include <linux/mm.h>
  24 #include <linux/fs.h>
  25 #include <linux/module.h>
  26 #include <linux/notifier.h>
  27 #include <linux/numa.h>
  28 #include <linux/oprofile.h>
  29 #include <linux/slab.h>
  30 #include <linux/spinlock.h>
  31 #include "pr_util.h"
  32
  33 #define RELEASE_ALL 9999
  34
  35 static DEFINE_SPINLOCK(buffer_lock);
  36 static DEFINE_SPINLOCK(cache_lock);
  37 static int num_spu_nodes;
  38 int spu_prof_num_nodes;
  39
  40 struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE];
  41 struct delayed_work spu_work;
  42 static unsigned max_spu_buff;
  43
  44 static void spu_buff_add(unsigned long int value, int spu)
  45 {
  46         /* spu buff is a circular buffer.  Add entries to the
  47          * head.  Head is the index to store the next value.
  48          * The buffer is full when there is one available entry
  49          * in the queue, i.e. head and tail can't be equal.
  50          * That way we can tell the difference between the
  51          * buffer being full versus empty.
  52          *
  53          *  ASSUPTION: the buffer_lock is held when this function
  54          *             is called to lock the buffer, head and tail.
  55          */
  56         int full = 1;
  57
  58         if (spu_buff[spu].head >= spu_buff[spu].tail) {
  59                 if ((spu_buff[spu].head - spu_buff[spu].tail)
  60                     <  (max_spu_buff - 1))
  61                         full = 0;
  62
  63         } else if (spu_buff[spu].tail > spu_buff[spu].head) {
  64                 if ((spu_buff[spu].tail - spu_buff[spu].head)
  65                     > 1)
  66                         full = 0;
  67         }
  68
  69         if (!full) {
  70                 spu_buff[spu].buff[spu_buff[spu].head] = value;
  71                 spu_buff[spu].head++;
  72
  73                 if (spu_buff[spu].head >= max_spu_buff)
  74                         spu_buff[spu].head = 0;
  75         } else {
  76                 /* From the user's perspective make the SPU buffer
  77                  * size management/overflow look like we are using
  78                  * per cpu buffers.  The user uses the same
  79                  * per cpu parameter to adjust the SPU buffer size.
  80                  * Increment the sample_lost_overflow to inform
  81                  * the user the buffer size needs to be increased.
  82                  */
  83                 oprofile_cpu_buffer_inc_smpl_lost();
  84         }
  85 }
  86
  87 /* This function copies the per SPU buffers to the
  88  * OProfile kernel buffer.
  89  */
  90 void sync_spu_buff(void)
  91 {
  92         int spu;
  93         unsigned long flags;
  94         int curr_head;
  95
  96         for (spu = 0; spu < num_spu_nodes; spu++) {
  97                 /* In case there was an issue and the buffer didn't
  98                  * get created skip it.
  99                  */
 100                 if (spu_buff[spu].buff == NULL)
 101                         continue;
 102
 103                 /* Hold the lock to make sure the head/tail
 104                  * doesn't change while spu_buff_add() is
 105                  * deciding if the buffer is full or not.
 106                  * Being a little paranoid.
 107                  */
 108                 spin_lock_irqsave(&buffer_lock, flags);
 109                 curr_head = spu_buff[spu].head;
 110                 spin_unlock_irqrestore(&buffer_lock, flags);
 111
 112                 /* Transfer the current contents to the kernel buffer.
 113                  * data can still be added to the head of the buffer.
 114                  */
 115                 oprofile_put_buff(spu_buff[spu].buff,
 116                                   spu_buff[spu].tail,
 117                                   curr_head, max_spu_buff);
 118
 119                 spin_lock_irqsave(&buffer_lock, flags);
 120                 spu_buff[spu].tail = curr_head;
 121                 spin_unlock_irqrestore(&buffer_lock, flags);
 122         }
 123
 124 }
 125
 126 static void wq_sync_spu_buff(struct work_struct *work)
 127 {
 128         /* move data from spu buffers to kernel buffer */
 129         sync_spu_buff();
 130
 131         /* only reschedule if profiling is not done */
 132         if (spu_prof_running)
 133                 schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE);
 134 }
 135
 136 /* Container for caching information about an active SPU task. */
 137 struct cached_info {
 138         struct vma_to_fileoffset_map *map;
 139         struct spu *the_spu;    /* needed to access pointer to local_store */
 140         struct kref cache_ref;
 141 };
 142
 143 static struct cached_info *spu_info[MAX_NUMNODES * 8];
 144
 145 static void destroy_cached_info(struct kref *kref)
 146 {
 147         struct cached_info *info;
 148
 149         info = container_of(kref, struct cached_info, cache_ref);
 150         vma_map_free(info->map);
 151         kfree(info);
 152         module_put(THIS_MODULE);
 153 }
 154
 155 /* Return the cached_info for the passed SPU number.
 156  * ATTENTION:  Callers are responsible for obtaining the
 157  *             cache_lock if needed prior to invoking this function.
 158  */
 159 static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num)
 160 {
 161         struct kref *ref;
 162         struct cached_info *ret_info;
 163
 164         if (spu_num >= num_spu_nodes) {
 165                 printk(KERN_ERR "SPU_PROF: "
 166                        "%s, line %d: Invalid index %d into spu info cache\n",
 167                        __func__, __LINE__, spu_num);
 168                 ret_info = NULL;
 169                 goto out;
 170         }
 171         if (!spu_info[spu_num] && the_spu) {
 172                 ref = spu_get_profile_private_kref(the_spu->ctx);
 173                 if (ref) {
 174                         spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
 175                         kref_get(&spu_info[spu_num]->cache_ref);
 176                 }
 177         }
 178
 179         ret_info = spu_info[spu_num];
 180  out:
 181         return ret_info;
 182 }
 183
 184
 185 /* Looks for cached info for the passed spu.  If not found, the
 186  * cached info is created for the passed spu.
 187  * Returns 0 for success; otherwise, -1 for error.
 188  */
 189 static int
 190 prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
 191 {
 192         unsigned long flags;
 193         struct vma_to_fileoffset_map *new_map;
 194         int retval = 0;
 195         struct cached_info *info;
 196
 197         /* We won't bother getting cache_lock here since
 198          * don't do anything with the cached_info that's returned.
 199          */
 200         info = get_cached_info(spu, spu->number);
 201
 202         if (info) {
 203                 pr_debug("Found cached SPU info.\n");
 204                 goto out;
 205         }
 206
 207         /* Create cached_info and set spu_info[spu->number] to point to it.
 208          * spu->number is a system-wide value, not a per-node value.
 209          */
 210         info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
 211         if (!info) {
 212                 printk(KERN_ERR "SPU_PROF: "
 213                        "%s, line %d: create vma_map failed\n",
 214                        __func__, __LINE__);
 215                 retval = -ENOMEM;
 216                 goto err_alloc;
 217         }
 218         new_map = create_vma_map(spu, objectId);
 219         if (!new_map) {
 220                 printk(KERN_ERR "SPU_PROF: "
 221                        "%s, line %d: create vma_map failed\n",
 222                        __func__, __LINE__);
 223                 retval = -ENOMEM;
 224                 goto err_alloc;
 225         }
 226
 227         pr_debug("Created vma_map\n");
 228         info->map = new_map;
 229         info->the_spu = spu;
 230         kref_init(&info->cache_ref);
 231         spin_lock_irqsave(&cache_lock, flags);
 232         spu_info[spu->number] = info;
 233         /* Increment count before passing off ref to SPUFS. */
 234         kref_get(&info->cache_ref);
 235
 236         /* We increment the module refcount here since SPUFS is
 237          * responsible for the final destruction of the cached_info,
 238          * and it must be able to access the destroy_cached_info()
 239          * function defined in the OProfile module.  We decrement
 240          * the module refcount in destroy_cached_info.
 241          */
 242         try_module_get(THIS_MODULE);
 243         spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
 244                                 destroy_cached_info);
 245         spin_unlock_irqrestore(&cache_lock, flags);
 246         goto out;
 247
 248 err_alloc:
 249         kfree(info);
 250 out:
 251         return retval;
 252 }
 253
 254 /*
 255  * NOTE:  The caller is responsible for locking the
 256  *        cache_lock prior to calling this function.
 257  */
 258 static int release_cached_info(int spu_index)
 259 {
 260         int index, end;
 261
 262         if (spu_index == RELEASE_ALL) {
 263                 end = num_spu_nodes;
 264                 index = 0;
 265         } else {
 266                 if (spu_index >= num_spu_nodes) {
 267                         printk(KERN_ERR "SPU_PROF: "
 268                                 "%s, line %d: "
 269                                 "Invalid index %d into spu info cache\n",
 270                                 __func__, __LINE__, spu_index);
 271                         goto out;
 272                 }
 273                 end = spu_index + 1;
 274                 index = spu_index;
 275         }
 276         for (; index < end; index++) {
 277                 if (spu_info[index]) {
 278                         kref_put(&spu_info[index]->cache_ref,
 279                                  destroy_cached_info);
 280                         spu_info[index] = NULL;
 281                 }
 282         }
 283
 284 out:
 285         return 0;
 286 }
 287
 288 /* The source code for fast_get_dcookie was "borrowed"
 289  * from drivers/oprofile/buffer_sync.c.
 290  */
 291
 292 /* Optimisation. We can manage without taking the dcookie sem
 293  * because we cannot reach this code without at least one
 294  * dcookie user still being registered (namely, the reader
 295  * of the event buffer).
 296  */
 297 static inline unsigned long fast_get_dcookie(struct path *path)
 298 {
 299         unsigned long cookie;
 300
 301         if (path->dentry->d_flags & DCACHE_COOKIE)
 302                 return (unsigned long)path->dentry;
 303         get_dcookie(path, &cookie);
 304         return cookie;
 305 }
 306
 307 /* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
 308  * which corresponds loosely to "application name". Also, determine
 309  * the offset for the SPU ELF object.  If computed offset is
 310  * non-zero, it implies an embedded SPU object; otherwise, it's a
 311  * separate SPU binary, in which case we retrieve it's dcookie.
 312  * For the embedded case, we must determine if SPU ELF is embedded
 313  * in the executable application or another file (i.e., shared lib).
 314  * If embedded in a shared lib, we must get the dcookie and return
 315  * that to the caller.
 316  */
 317 static unsigned long
 318 get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp,
 319                             unsigned long *spu_bin_dcookie,
 320                             unsigned long spu_ref)
 321 {
 322         unsigned long app_cookie = 0;
 323         unsigned int my_offset = 0;
 324         struct file *app = NULL;
 325         struct vm_area_struct *vma;
 326         struct mm_struct *mm = spu->mm;
 327
 328         if (!mm)
 329                 goto out;
 330
 331         down_read(&mm->mmap_sem);
 332
 333         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 334                 if (!vma->vm_file)
 335                         continue;
 336                 if (!(vma->vm_flags & VM_EXECUTABLE))
 337                         continue;
 338                 app_cookie = fast_get_dcookie(&vma->vm_file->f_path);
 339                 pr_debug("got dcookie for %s\n",
 340                          vma->vm_file->f_dentry->d_name.name);
 341                 app = vma->vm_file;
 342                 break;
 343         }
 344
 345         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 346                 if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
 347                         continue;
 348                 my_offset = spu_ref - vma->vm_start;
 349                 if (!vma->vm_file)
 350                         goto fail_no_image_cookie;
 351
 352                 pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
 353                          my_offset, spu_ref,
 354                          vma->vm_file->f_dentry->d_name.name);
 355                 *offsetp = my_offset;
 356                 break;
 357         }
 358
 359         *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
 360         pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name);
 361
 362         up_read(&mm->mmap_sem);
 363
 364 out:
 365         return app_cookie;
 366
 367 fail_no_image_cookie:
 368         up_read(&mm->mmap_sem);
 369
 370         printk(KERN_ERR "SPU_PROF: "
 371                 "%s, line %d: Cannot find dcookie for SPU binary\n",
 372                 __func__, __LINE__);
 373         goto out;
 374 }
 375
 376
 377
 378 /* This function finds or creates cached context information for the
 379  * passed SPU and records SPU context information into the OProfile
 380  * event buffer.
 381  */
 382 static int process_context_switch(struct spu *spu, unsigned long objectId)
 383 {
 384         unsigned long flags;
 385         int retval;
 386         unsigned int offset = 0;
 387         unsigned long spu_cookie = 0, app_dcookie;
 388
 389         retval = prepare_cached_spu_info(spu, objectId);
 390         if (retval)
 391                 goto out;
 392
 393         /* Get dcookie first because a mutex_lock is taken in that
 394          * code path, so interrupts must not be disabled.
 395          */
 396         app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
 397         if (!app_dcookie || !spu_cookie) {
 398                 retval  = -ENOENT;
 399                 goto out;
 400         }
 401
 402         /* Record context info in event buffer */
 403         spin_lock_irqsave(&buffer_lock, flags);
 404         spu_buff_add(ESCAPE_CODE, spu->number);
 405         spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number);
 406         spu_buff_add(spu->number, spu->number);
 407         spu_buff_add(spu->pid, spu->number);
 408         spu_buff_add(spu->tgid, spu->number);
 409         spu_buff_add(app_dcookie, spu->number);
 410         spu_buff_add(spu_cookie, spu->number);
 411         spu_buff_add(offset, spu->number);
 412
 413         /* Set flag to indicate SPU PC data can now be written out.  If
 414          * the SPU program counter data is seen before an SPU context
 415          * record is seen, the postprocessing will fail.
 416          */
 417         spu_buff[spu->number].ctx_sw_seen = 1;
 418
 419         spin_unlock_irqrestore(&buffer_lock, flags);
 420         smp_wmb();      /* insure spu event buffer updates are written */
 421                         /* don't want entries intermingled... */
 422 out:
 423         return retval;
 424 }
 425
 426 /*
 427  * This function is invoked on either a bind_context or unbind_context.
 428  * If called for an unbind_context, the val arg is 0; otherwise,
 429  * it is the object-id value for the spu context.
 430  * The data arg is of type 'struct spu *'.
 431  */
 432 static int spu_active_notify(struct notifier_block *self, unsigned long val,
 433                                 void *data)
 434 {
 435         int retval;
 436         unsigned long flags;
 437         struct spu *the_spu = data;
 438
 439         pr_debug("SPU event notification arrived\n");
 440         if (!val) {
 441                 spin_lock_irqsave(&cache_lock, flags);
 442                 retval = release_cached_info(the_spu->number);
 443                 spin_unlock_irqrestore(&cache_lock, flags);
 444         } else {
 445                 retval = process_context_switch(the_spu, val);
 446         }
 447         return retval;
 448 }
 449
 450 static struct notifier_block spu_active = {
 451         .notifier_call = spu_active_notify,
 452 };
 453
 454 static int number_of_online_nodes(void)
 455 {
 456         u32 cpu; u32 tmp;
 457         int nodes = 0;
 458         for_each_online_cpu(cpu) {
 459                 tmp = cbe_cpu_to_node(cpu) + 1;
 460                 if (tmp > nodes)
 461                         nodes++;
 462         }
 463         return nodes;
 464 }
 465
 466 static int oprofile_spu_buff_create(void)
 467 {
 468         int spu;
 469
 470         max_spu_buff = oprofile_get_cpu_buffer_size();
 471
 472         for (spu = 0; spu < num_spu_nodes; spu++) {
 473                 /* create circular buffers to store the data in.
 474                  * use locks to manage accessing the buffers
 475                  */
 476                 spu_buff[spu].head = 0;
 477                 spu_buff[spu].tail = 0;
 478
 479                 /*
 480                  * Create a buffer for each SPU.  Can't reliably
 481                  * create a single buffer for all spus due to not
 482                  * enough contiguous kernel memory.
 483                  */
 484
 485                 spu_buff[spu].buff = kzalloc((max_spu_buff
 486                                               * sizeof(unsigned long)),
 487                                              GFP_KERNEL);
 488
 489                 if (!spu_buff[spu].buff) {
 490                         printk(KERN_ERR "SPU_PROF: "
 491                                "%s, line %d:  oprofile_spu_buff_create "
 492                        "failed to allocate spu buffer %d.\n",
 493                                __func__, __LINE__, spu);
 494
 495                         /* release the spu buffers that have been allocated */
 496                         while (spu >= 0) {
 497                                 kfree(spu_buff[spu].buff);
 498                                 spu_buff[spu].buff = 0;
 499                                 spu--;
 500                         }
 501                         return -ENOMEM;
 502                 }
 503         }
 504         return 0;
 505 }
 506
 507 /* The main purpose of this function is to synchronize
 508  * OProfile with SPUFS by registering to be notified of
 509  * SPU task switches.
 510  *
 511  * NOTE: When profiling SPUs, we must ensure that only
 512  * spu_sync_start is invoked and not the generic sync_start
 513  * in drivers/oprofile/oprof.c.  A return value of
 514  * SKIP_GENERIC_SYNC or SYNC_START_ERROR will
 515  * accomplish this.
 516  */
 517 int spu_sync_start(void)
 518 {
 519         int spu;
 520         int ret = SKIP_GENERIC_SYNC;
 521         int register_ret;
 522         unsigned long flags = 0;
 523
 524         spu_prof_num_nodes = number_of_online_nodes();
 525         num_spu_nodes = spu_prof_num_nodes * 8;
 526         INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff);
 527
 528         /* create buffer for storing the SPU data to put in
 529          * the kernel buffer.
 530          */
 531         ret = oprofile_spu_buff_create();
 532         if (ret)
 533                 goto out;
 534
 535         spin_lock_irqsave(&buffer_lock, flags);
 536         for (spu = 0; spu < num_spu_nodes; spu++) {
 537                 spu_buff_add(ESCAPE_CODE, spu);
 538                 spu_buff_add(SPU_PROFILING_CODE, spu);
 539                 spu_buff_add(num_spu_nodes, spu);
 540         }
 541         spin_unlock_irqrestore(&buffer_lock, flags);
 542
 543         for (spu = 0; spu < num_spu_nodes; spu++) {
 544                 spu_buff[spu].ctx_sw_seen = 0;
 545                 spu_buff[spu].last_guard_val = 0;
 546         }
 547
 548         /* Register for SPU events  */
 549         register_ret = spu_switch_event_register(&spu_active);
 550         if (register_ret) {
 551                 ret = SYNC_START_ERROR;
 552                 goto out;
 553         }
 554
 555         pr_debug("spu_sync_start -- running.\n");
 556 out:
 557         return ret;
 558 }
 559
 560 /* Record SPU program counter samples to the oprofile event buffer. */
 561 void spu_sync_buffer(int spu_num, unsigned int *samples,
 562                      int num_samples)
 563 {
 564         unsigned long long file_offset;
 565         unsigned long flags;
 566         int i;
 567         struct vma_to_fileoffset_map *map;
 568         struct spu *the_spu;
 569         unsigned long long spu_num_ll = spu_num;
 570         unsigned long long spu_num_shifted = spu_num_ll << 32;
 571         struct cached_info *c_info;
 572
 573         /* We need to obtain the cache_lock here because it's
 574          * possible that after getting the cached_info, the SPU job
 575          * corresponding to this cached_info may end, thus resulting
 576          * in the destruction of the cached_info.
 577          */
 578         spin_lock_irqsave(&cache_lock, flags);
 579         c_info = get_cached_info(NULL, spu_num);
 580         if (!c_info) {
 581                 /* This legitimately happens when the SPU task ends before all
 582                  * samples are recorded.
 583                  * No big deal -- so we just drop a few samples.
 584                  */
 585                 pr_debug("SPU_PROF: No cached SPU contex "
 586                           "for SPU #%d. Dropping samples.\n", spu_num);
 587                 goto out;
 588         }
 589
 590         map = c_info->map;
 591         the_spu = c_info->the_spu;
 592         spin_lock(&buffer_lock);
 593         for (i = 0; i < num_samples; i++) {
 594                 unsigned int sample = *(samples+i);
 595                 int grd_val = 0;
 596                 file_offset = 0;
 597                 if (sample == 0)
 598                         continue;
 599                 file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);
 600
 601                 /* If overlays are used by this SPU application, the guard
 602                  * value is non-zero, indicating which overlay section is in
 603                  * use.  We need to discard samples taken during the time
 604                  * period which an overlay occurs (i.e., guard value changes).
 605                  */
 606                 if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) {
 607                         spu_buff[spu_num].last_guard_val = grd_val;
 608                         /* Drop the rest of the samples. */
 609                         break;
 610                 }
 611
 612                 /* We must ensure that the SPU context switch has been written
 613                  * out before samples for the SPU.  Otherwise, the SPU context
 614                  * information is not available and the postprocessing of the
 615                  * SPU PC will fail with no available anonymous map information.
 616                  */
 617                 if (spu_buff[spu_num].ctx_sw_seen)
 618                         spu_buff_add((file_offset | spu_num_shifted),
 619                                          spu_num);
 620         }
 621         spin_unlock(&buffer_lock);
 622 out:
 623         spin_unlock_irqrestore(&cache_lock, flags);
 624 }
 625
 626
 627 int spu_sync_stop(void)
 628 {
 629         unsigned long flags = 0;
 630         int ret;
 631         int k;
 632
 633         ret = spu_switch_event_unregister(&spu_active);
 634
 635         if (ret)
 636                 printk(KERN_ERR "SPU_PROF: "
 637                        "%s, line %d: spu_switch_event_unregister "      \
 638                        "returned %d\n",
 639                        __func__, __LINE__, ret);
 640
 641         /* flush any remaining data in the per SPU buffers */
 642         sync_spu_buff();
 643
 644         spin_lock_irqsave(&cache_lock, flags);
 645         ret = release_cached_info(RELEASE_ALL);
 646         spin_unlock_irqrestore(&cache_lock, flags);
 647
 648         /* remove scheduled work queue item rather then waiting
 649          * for every queued entry to execute.  Then flush pending
 650          * system wide buffer to event buffer.
 651          */
 652         cancel_delayed_work(&spu_work);
 653
 654         for (k = 0; k < num_spu_nodes; k++) {
 655                 spu_buff[k].ctx_sw_seen = 0;
 656
 657                 /*
 658                  * spu_sys_buff will be null if there was a problem
 659                  * allocating the buffer.  Only delete if it exists.
 660                  */
 661                 kfree(spu_buff[k].buff);
 662                 spu_buff[k].buff = 0;
 663         }
 664         pr_debug("spu_sync_stop -- done.\n");
 665         return ret;
 666 }
 667