drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include "drmP.h"
  28 #include "drm.h"
  29
  30 #include "nouveau_drv.h"
  31 #include "nouveau_ramht.h"
  32 #include "nouveau_dma.h"
  33
  34 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
  35 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
  36
  37 struct nouveau_fence {
  38         struct nouveau_channel *channel;
  39         struct kref refcount;
  40         struct list_head entry;
  41
  42         uint32_t sequence;
  43         bool signalled;
  44
  45         void (*work)(void *priv, bool signalled);
  46         void *priv;
  47 };
  48
  49 struct nouveau_semaphore {
  50         struct kref ref;
  51         struct drm_device *dev;
  52         struct drm_mm_node *mem;
  53 };
  54
  55 static inline struct nouveau_fence *
  56 nouveau_fence(void *sync_obj)
  57 {
  58         return (struct nouveau_fence *)sync_obj;
  59 }
  60
  61 static void
  62 nouveau_fence_del(struct kref *ref)
  63 {
  64         struct nouveau_fence *fence =
  65                 container_of(ref, struct nouveau_fence, refcount);
  66
  67         kfree(fence);
  68 }
  69
  70 void
  71 nouveau_fence_update(struct nouveau_channel *chan)
  72 {
  73         struct drm_device *dev = chan->dev;
  74         struct nouveau_fence *tmp, *fence;
  75         uint32_t sequence;
  76
  77         spin_lock(&chan->fence.lock);
  78
  79         if (USE_REFCNT(dev))
  80                 sequence = nvchan_rd32(chan, 0x48);
  81         else
  82                 sequence = atomic_read(&chan->fence.last_sequence_irq);
  83
  84         if (chan->fence.sequence_ack == sequence)
  85                 goto out;
  86         chan->fence.sequence_ack = sequence;
  87
  88         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
  89                 sequence = fence->sequence;
  90                 fence->signalled = true;
  91                 list_del(&fence->entry);
  92
  93                 if (unlikely(fence->work))
  94                         fence->work(fence->priv, true);
  95
  96                 kref_put(&fence->refcount, nouveau_fence_del);
  97
  98                 if (sequence == chan->fence.sequence_ack)
  99                         break;
 100         }
 101 out:
 102         spin_unlock(&chan->fence.lock);
 103 }
 104
 105 int
 106 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
 107                   bool emit)
 108 {
 109         struct nouveau_fence *fence;
 110         int ret = 0;
 111
 112         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 113         if (!fence)
 114                 return -ENOMEM;
 115         kref_init(&fence->refcount);
 116         fence->channel = chan;
 117
 118         if (emit)
 119                 ret = nouveau_fence_emit(fence);
 120
 121         if (ret)
 122                 nouveau_fence_unref((void *)&fence);
 123         *pfence = fence;
 124         return ret;
 125 }
 126
 127 struct nouveau_channel *
 128 nouveau_fence_channel(struct nouveau_fence *fence)
 129 {
 130         return fence ? fence->channel : NULL;
 131 }
 132
 133 int
 134 nouveau_fence_emit(struct nouveau_fence *fence)
 135 {
 136         struct nouveau_channel *chan = fence->channel;
 137         struct drm_device *dev = chan->dev;
 138         int ret;
 139
 140         ret = RING_SPACE(chan, 2);
 141         if (ret)
 142                 return ret;
 143
 144         if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
 145                 nouveau_fence_update(chan);
 146
 147                 BUG_ON(chan->fence.sequence ==
 148                        chan->fence.sequence_ack - 1);
 149         }
 150
 151         fence->sequence = ++chan->fence.sequence;
 152
 153         kref_get(&fence->refcount);
 154         spin_lock(&chan->fence.lock);
 155         list_add_tail(&fence->entry, &chan->fence.pending);
 156         spin_unlock(&chan->fence.lock);
 157
 158         BEGIN_RING(chan, NvSubSw, USE_REFCNT(dev) ? 0x0050 : 0x0150, 1);
 159         OUT_RING(chan, fence->sequence);
 160         FIRE_RING(chan);
 161
 162         return 0;
 163 }
 164
 165 void
 166 nouveau_fence_work(struct nouveau_fence *fence,
 167                    void (*work)(void *priv, bool signalled),
 168                    void *priv)
 169 {
 170         BUG_ON(fence->work);
 171
 172         spin_lock(&fence->channel->fence.lock);
 173
 174         if (fence->signalled) {
 175                 work(priv, true);
 176         } else {
 177                 fence->work = work;
 178                 fence->priv = priv;
 179         }
 180
 181         spin_unlock(&fence->channel->fence.lock);
 182 }
 183
 184 void
 185 nouveau_fence_unref(void **sync_obj)
 186 {
 187         struct nouveau_fence *fence = nouveau_fence(*sync_obj);
 188
 189         if (fence)
 190                 kref_put(&fence->refcount, nouveau_fence_del);
 191         *sync_obj = NULL;
 192 }
 193
 194 void *
 195 nouveau_fence_ref(void *sync_obj)
 196 {
 197         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 198
 199         kref_get(&fence->refcount);
 200         return sync_obj;
 201 }
 202
 203 bool
 204 nouveau_fence_signalled(void *sync_obj, void *sync_arg)
 205 {
 206         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 207         struct nouveau_channel *chan = fence->channel;
 208
 209         if (fence->signalled)
 210                 return true;
 211
 212         nouveau_fence_update(chan);
 213         return fence->signalled;
 214 }
 215
 216 int
 217 nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
 218 {
 219         unsigned long timeout = jiffies + (3 * DRM_HZ);
 220         int ret = 0;
 221
 222         while (1) {
 223                 if (nouveau_fence_signalled(sync_obj, sync_arg))
 224                         break;
 225
 226                 if (time_after_eq(jiffies, timeout)) {
 227                         ret = -EBUSY;
 228                         break;
 229                 }
 230
 231                 __set_current_state(intr ? TASK_INTERRUPTIBLE
 232                         : TASK_UNINTERRUPTIBLE);
 233                 if (lazy)
 234                         schedule_timeout(1);
 235
 236                 if (intr && signal_pending(current)) {
 237                         ret = -ERESTARTSYS;
 238                         break;
 239                 }
 240         }
 241
 242         __set_current_state(TASK_RUNNING);
 243
 244         return ret;
 245 }
 246
 247 static struct nouveau_semaphore *
 248 alloc_semaphore(struct drm_device *dev)
 249 {
 250         struct drm_nouveau_private *dev_priv = dev->dev_private;
 251         struct nouveau_semaphore *sema;
 252
 253         if (!USE_SEMA(dev))
 254                 return NULL;
 255
 256         sema = kmalloc(sizeof(*sema), GFP_KERNEL);
 257         if (!sema)
 258                 goto fail;
 259
 260         spin_lock(&dev_priv->fence.lock);
 261         sema->mem = drm_mm_search_free(&dev_priv->fence.heap, 4, 0, 0);
 262         if (sema->mem)
 263                 sema->mem = drm_mm_get_block(sema->mem, 4, 0);
 264         spin_unlock(&dev_priv->fence.lock);
 265
 266         if (!sema->mem)
 267                 goto fail;
 268
 269         kref_init(&sema->ref);
 270         sema->dev = dev;
 271         nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 0);
 272
 273         return sema;
 274 fail:
 275         kfree(sema);
 276         return NULL;
 277 }
 278
 279 static void
 280 free_semaphore(struct kref *ref)
 281 {
 282         struct nouveau_semaphore *sema =
 283                 container_of(ref, struct nouveau_semaphore, ref);
 284         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 285
 286         spin_lock(&dev_priv->fence.lock);
 287         drm_mm_put_block(sema->mem);
 288         spin_unlock(&dev_priv->fence.lock);
 289
 290         kfree(sema);
 291 }
 292
 293 static void
 294 semaphore_work(void *priv, bool signalled)
 295 {
 296         struct nouveau_semaphore *sema = priv;
 297         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 298
 299         if (unlikely(!signalled))
 300                 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
 301
 302         kref_put(&sema->ref, free_semaphore);
 303 }
 304
 305 static int
 306 emit_semaphore(struct nouveau_channel *chan, int method,
 307                struct nouveau_semaphore *sema)
 308 {
 309         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 310         struct nouveau_fence *fence;
 311         bool smart = (dev_priv->card_type >= NV_50);
 312         int ret;
 313
 314         ret = RING_SPACE(chan, smart ? 8 : 4);
 315         if (ret)
 316                 return ret;
 317
 318         if (smart) {
 319                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 320                 OUT_RING(chan, NvSema);
 321         }
 322         BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 1);
 323         OUT_RING(chan, sema->mem->start);
 324
 325         if (smart && method == NV_SW_SEMAPHORE_ACQUIRE) {
 326                 /*
 327                  * NV50 tries to be too smart and context-switch
 328                  * between semaphores instead of doing a "first come,
 329                  * first served" strategy like previous cards
 330                  * do.
 331                  *
 332                  * That's bad because the ACQUIRE latency can get as
 333                  * large as the PFIFO context time slice in the
 334                  * typical DRI2 case where you have several
 335                  * outstanding semaphores at the same moment.
 336                  *
 337                  * If we're going to ACQUIRE, force the card to
 338                  * context switch before, just in case the matching
 339                  * RELEASE is already scheduled to be executed in
 340                  * another channel.
 341                  */
 342                 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
 343                 OUT_RING(chan, 0);
 344         }
 345
 346         BEGIN_RING(chan, NvSubSw, method, 1);
 347         OUT_RING(chan, 1);
 348
 349         if (smart && method == NV_SW_SEMAPHORE_RELEASE) {
 350                 /*
 351                  * Force the card to context switch, there may be
 352                  * another channel waiting for the semaphore we just
 353                  * released.
 354                  */
 355                 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
 356                 OUT_RING(chan, 0);
 357         }
 358
 359         /* Delay semaphore destruction until its work is done */
 360         ret = nouveau_fence_new(chan, &fence, true);
 361         if (ret)
 362                 return ret;
 363
 364         kref_get(&sema->ref);
 365         nouveau_fence_work(fence, semaphore_work, sema);
 366         nouveau_fence_unref((void *)&fence);
 367
 368         return 0;
 369 }
 370
 371 int
 372 nouveau_fence_sync(struct nouveau_fence *fence,
 373                    struct nouveau_channel *wchan)
 374 {
 375         struct nouveau_channel *chan = nouveau_fence_channel(fence);
 376         struct drm_device *dev = wchan->dev;
 377         struct nouveau_semaphore *sema;
 378         int ret;
 379
 380         if (likely(!fence || chan == wchan ||
 381                    nouveau_fence_signalled(fence, NULL)))
 382                 return 0;
 383
 384         sema = alloc_semaphore(dev);
 385         if (!sema) {
 386                 /* Early card or broken userspace, fall back to
 387                  * software sync. */
 388                 return nouveau_fence_wait(fence, NULL, false, false);
 389         }
 390
 391         /* Make wchan wait until it gets signalled */
 392         ret = emit_semaphore(wchan, NV_SW_SEMAPHORE_ACQUIRE, sema);
 393         if (ret)
 394                 goto out;
 395
 396         /* Signal the semaphore from chan */
 397         ret = emit_semaphore(chan, NV_SW_SEMAPHORE_RELEASE, sema);
 398 out:
 399         kref_put(&sema->ref, free_semaphore);
 400         return ret;
 401 }
 402
 403 int
 404 nouveau_fence_flush(void *sync_obj, void *sync_arg)
 405 {
 406         return 0;
 407 }
 408
 409 int
 410 nouveau_fence_channel_init(struct nouveau_channel *chan)
 411 {
 412         struct drm_device *dev = chan->dev;
 413         struct drm_nouveau_private *dev_priv = dev->dev_private;
 414         struct nouveau_gpuobj *obj = NULL;
 415         int ret;
 416
 417         /* Create an NV_SW object for various sync purposes */
 418         ret = nouveau_gpuobj_sw_new(chan, NV_SW, &obj);
 419         if (ret)
 420                 return ret;
 421
 422         ret = nouveau_ramht_insert(chan, NvSw, obj);
 423         nouveau_gpuobj_ref(NULL, &obj);
 424         if (ret)
 425                 return ret;
 426
 427         ret = RING_SPACE(chan, 2);
 428         if (ret)
 429                 return ret;
 430         BEGIN_RING(chan, NvSubSw, 0, 1);
 431         OUT_RING(chan, NvSw);
 432
 433         /* Create a DMA object for the shared cross-channel sync area. */
 434         if (USE_SEMA(dev)) {
 435                 struct drm_mm_node *mem = dev_priv->fence.bo->bo.mem.mm_node;
 436
 437                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 438                                              mem->start << PAGE_SHIFT,
 439                                              mem->size << PAGE_SHIFT,
 440                                              NV_DMA_ACCESS_RW,
 441                                              NV_DMA_TARGET_VIDMEM, &obj);
 442                 if (ret)
 443                         return ret;
 444
 445                 ret = nouveau_ramht_insert(chan, NvSema, obj);
 446                 nouveau_gpuobj_ref(NULL, &obj);
 447                 if (ret)
 448                         return ret;
 449
 450                 ret = RING_SPACE(chan, 2);
 451                 if (ret)
 452                         return ret;
 453                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 454                 OUT_RING(chan, NvSema);
 455         }
 456
 457         FIRE_RING(chan);
 458
 459         INIT_LIST_HEAD(&chan->fence.pending);
 460         spin_lock_init(&chan->fence.lock);
 461         atomic_set(&chan->fence.last_sequence_irq, 0);
 462
 463         return 0;
 464 }
 465
 466 void
 467 nouveau_fence_channel_fini(struct nouveau_channel *chan)
 468 {
 469         struct nouveau_fence *tmp, *fence;
 470
 471         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
 472                 fence->signalled = true;
 473                 list_del(&fence->entry);
 474
 475                 if (unlikely(fence->work))
 476                         fence->work(fence->priv, false);
 477
 478                 kref_put(&fence->refcount, nouveau_fence_del);
 479         }
 480 }
 481
 482 int
 483 nouveau_fence_init(struct drm_device *dev)
 484 {
 485         struct drm_nouveau_private *dev_priv = dev->dev_private;
 486         int ret;
 487
 488         /* Create a shared VRAM heap for cross-channel sync. */
 489         if (USE_SEMA(dev)) {
 490                 ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM,
 491                                      0, 0, false, true, &dev_priv->fence.bo);
 492                 if (ret)
 493                         return ret;
 494
 495                 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
 496                 if (ret)
 497                         goto fail;
 498
 499                 ret = nouveau_bo_map(dev_priv->fence.bo);
 500                 if (ret)
 501                         goto fail;
 502
 503                 ret = drm_mm_init(&dev_priv->fence.heap, 0,
 504                                   dev_priv->fence.bo->bo.mem.size);
 505                 if (ret)
 506                         goto fail;
 507
 508                 spin_lock_init(&dev_priv->fence.lock);
 509         }
 510
 511         return 0;
 512 fail:
 513         nouveau_bo_unmap(dev_priv->fence.bo);
 514         nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 515         return ret;
 516 }
 517
 518 void
 519 nouveau_fence_fini(struct drm_device *dev)
 520 {
 521         struct drm_nouveau_private *dev_priv = dev->dev_private;
 522
 523         if (USE_SEMA(dev)) {
 524                 drm_mm_takedown(&dev_priv->fence.heap);
 525                 nouveau_bo_unmap(dev_priv->fence.bo);
 526                 nouveau_bo_unpin(dev_priv->fence.bo);
 527                 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 528         }
 529 }