drivers/gpu/drm/nouveau/nouveau_object.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drv.h"
  36 #include "nouveau_drm.h"
  37 #include "nouveau_ramht.h"
  38 #include "nouveau_vm.h"
  39 #include "nv50_display.h"
  40
  41 struct nouveau_gpuobj_method {
  42         struct list_head head;
  43         u32 mthd;
  44         int (*exec)(struct nouveau_channel *, u32 class, u32 mthd, u32 data);
  45 };
  46
  47 struct nouveau_gpuobj_class {
  48         struct list_head head;
  49         struct list_head methods;
  50         u32 id;
  51         u32 engine;
  52 };
  53
  54 int
  55 nouveau_gpuobj_class_new(struct drm_device *dev, u32 class, u32 engine)
  56 {
  57         struct drm_nouveau_private *dev_priv = dev->dev_private;
  58         struct nouveau_gpuobj_class *oc;
  59
  60         oc = kzalloc(sizeof(*oc), GFP_KERNEL);
  61         if (!oc)
  62                 return -ENOMEM;
  63
  64         INIT_LIST_HEAD(&oc->methods);
  65         oc->id = class;
  66         oc->engine = engine;
  67         list_add(&oc->head, &dev_priv->classes);
  68         return 0;
  69 }
  70
  71 int
  72 nouveau_gpuobj_mthd_new(struct drm_device *dev, u32 class, u32 mthd,
  73                         int (*exec)(struct nouveau_channel *, u32, u32, u32))
  74 {
  75         struct drm_nouveau_private *dev_priv = dev->dev_private;
  76         struct nouveau_gpuobj_method *om;
  77         struct nouveau_gpuobj_class *oc;
  78
  79         list_for_each_entry(oc, &dev_priv->classes, head) {
  80                 if (oc->id == class)
  81                         goto found;
  82         }
  83
  84         return -EINVAL;
  85
  86 found:
  87         om = kzalloc(sizeof(*om), GFP_KERNEL);
  88         if (!om)
  89                 return -ENOMEM;
  90
  91         om->mthd = mthd;
  92         om->exec = exec;
  93         list_add(&om->head, &oc->methods);
  94         return 0;
  95 }
  96
  97 int
  98 nouveau_gpuobj_mthd_call(struct nouveau_channel *chan,
  99                          u32 class, u32 mthd, u32 data)
 100 {
 101         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 102         struct nouveau_gpuobj_method *om;
 103         struct nouveau_gpuobj_class *oc;
 104
 105         list_for_each_entry(oc, &dev_priv->classes, head) {
 106                 if (oc->id != class)
 107                         continue;
 108
 109                 list_for_each_entry(om, &oc->methods, head) {
 110                         if (om->mthd == mthd)
 111                                 return om->exec(chan, class, mthd, data);
 112                 }
 113         }
 114
 115         return -ENOENT;
 116 }
 117
 118 int
 119 nouveau_gpuobj_mthd_call2(struct drm_device *dev, int chid,
 120                           u32 class, u32 mthd, u32 data)
 121 {
 122         struct drm_nouveau_private *dev_priv = dev->dev_private;
 123         struct nouveau_channel *chan = NULL;
 124         unsigned long flags;
 125         int ret = -EINVAL;
 126
 127         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 128         if (chid >= 0 && chid < dev_priv->engine.fifo.channels)
 129                 chan = dev_priv->channels.ptr[chid];
 130         if (chan)
 131                 ret = nouveau_gpuobj_mthd_call(chan, class, mthd, data);
 132         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 133         return ret;
 134 }
 135
 136 /* NVidia uses context objects to drive drawing operations.
 137
 138    Context objects can be selected into 8 subchannels in the FIFO,
 139    and then used via DMA command buffers.
 140
 141    A context object is referenced by a user defined handle (CARD32). The HW
 142    looks up graphics objects in a hash table in the instance RAM.
 143
 144    An entry in the hash table consists of 2 CARD32. The first CARD32 contains
 145    the handle, the second one a bitfield, that contains the address of the
 146    object in instance RAM.
 147
 148    The format of the second CARD32 seems to be:
 149
 150    NV4 to NV30:
 151
 152    15: 0  instance_addr >> 4
 153    17:16  engine (here uses 1 = graphics)
 154    28:24  channel id (here uses 0)
 155    31     valid (use 1)
 156
 157    NV40:
 158
 159    15: 0  instance_addr >> 4   (maybe 19-0)
 160    21:20  engine (here uses 1 = graphics)
 161    I'm unsure about the other bits, but using 0 seems to work.
 162
 163    The key into the hash table depends on the object handle and channel id and
 164    is given as:
 165 */
 166
 167 int
 168 nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
 169                    uint32_t size, int align, uint32_t flags,
 170                    struct nouveau_gpuobj **gpuobj_ret)
 171 {
 172         struct drm_nouveau_private *dev_priv = dev->dev_private;
 173         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 174         struct nouveau_gpuobj *gpuobj;
 175         struct drm_mm_node *ramin = NULL;
 176         int ret, i;
 177
 178         NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n",
 179                  chan ? chan->id : -1, size, align, flags);
 180
 181         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 182         if (!gpuobj)
 183                 return -ENOMEM;
 184         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 185         gpuobj->dev = dev;
 186         gpuobj->flags = flags;
 187         kref_init(&gpuobj->refcount);
 188         gpuobj->size = size;
 189
 190         spin_lock(&dev_priv->ramin_lock);
 191         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 192         spin_unlock(&dev_priv->ramin_lock);
 193
 194         if (!(flags & NVOBJ_FLAG_VM) && chan) {
 195                 ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
 196                 if (ramin)
 197                         ramin = drm_mm_get_block(ramin, size, align);
 198                 if (!ramin) {
 199                         nouveau_gpuobj_ref(NULL, &gpuobj);
 200                         return -ENOMEM;
 201                 }
 202
 203                 gpuobj->pinst = chan->ramin->pinst;
 204                 if (gpuobj->pinst != ~0)
 205                         gpuobj->pinst += ramin->start;
 206
 207                 gpuobj->cinst = ramin->start;
 208                 gpuobj->vinst = ramin->start + chan->ramin->vinst;
 209                 gpuobj->node  = ramin;
 210         } else {
 211                 ret = instmem->get(gpuobj, chan, size, align);
 212                 if (ret) {
 213                         nouveau_gpuobj_ref(NULL, &gpuobj);
 214                         return ret;
 215                 }
 216
 217                 ret = -ENOSYS;
 218                 if (!(flags & NVOBJ_FLAG_DONT_MAP))
 219                         ret = instmem->map(gpuobj);
 220                 if (ret)
 221                         gpuobj->pinst = ~0;
 222
 223                 gpuobj->cinst = NVOBJ_CINST_GLOBAL;
 224         }
 225
 226         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 227                 for (i = 0; i < gpuobj->size; i += 4)
 228                         nv_wo32(gpuobj, i, 0);
 229                 instmem->flush(dev);
 230         }
 231
 232
 233         *gpuobj_ret = gpuobj;
 234         return 0;
 235 }
 236
 237 int
 238 nouveau_gpuobj_init(struct drm_device *dev)
 239 {
 240         struct drm_nouveau_private *dev_priv = dev->dev_private;
 241
 242         NV_DEBUG(dev, "\n");
 243
 244         INIT_LIST_HEAD(&dev_priv->gpuobj_list);
 245         INIT_LIST_HEAD(&dev_priv->classes);
 246         spin_lock_init(&dev_priv->ramin_lock);
 247         dev_priv->ramin_base = ~0;
 248
 249         return 0;
 250 }
 251
 252 void
 253 nouveau_gpuobj_takedown(struct drm_device *dev)
 254 {
 255         struct drm_nouveau_private *dev_priv = dev->dev_private;
 256         struct nouveau_gpuobj_method *om, *tm;
 257         struct nouveau_gpuobj_class *oc, *tc;
 258
 259         NV_DEBUG(dev, "\n");
 260
 261         list_for_each_entry_safe(oc, tc, &dev_priv->classes, head) {
 262                 list_for_each_entry_safe(om, tm, &oc->methods, head) {
 263                         list_del(&om->head);
 264                         kfree(om);
 265                 }
 266                 list_del(&oc->head);
 267                 kfree(oc);
 268         }
 269
 270         BUG_ON(!list_empty(&dev_priv->gpuobj_list));
 271 }
 272
 273
 274 static void
 275 nouveau_gpuobj_del(struct kref *ref)
 276 {
 277         struct nouveau_gpuobj *gpuobj =
 278                 container_of(ref, struct nouveau_gpuobj, refcount);
 279         struct drm_device *dev = gpuobj->dev;
 280         struct drm_nouveau_private *dev_priv = dev->dev_private;
 281         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 282         int i;
 283
 284         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 285
 286         if (gpuobj->node && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
 287                 for (i = 0; i < gpuobj->size; i += 4)
 288                         nv_wo32(gpuobj, i, 0);
 289                 instmem->flush(dev);
 290         }
 291
 292         if (gpuobj->dtor)
 293                 gpuobj->dtor(dev, gpuobj);
 294
 295         if (gpuobj->cinst == NVOBJ_CINST_GLOBAL) {
 296                 if (gpuobj->node) {
 297                         instmem->unmap(gpuobj);
 298                         instmem->put(gpuobj);
 299                 }
 300         } else {
 301                 if (gpuobj->node) {
 302                         spin_lock(&dev_priv->ramin_lock);
 303                         drm_mm_put_block(gpuobj->node);
 304                         spin_unlock(&dev_priv->ramin_lock);
 305                 }
 306         }
 307
 308         spin_lock(&dev_priv->ramin_lock);
 309         list_del(&gpuobj->list);
 310         spin_unlock(&dev_priv->ramin_lock);
 311
 312         kfree(gpuobj);
 313 }
 314
 315 void
 316 nouveau_gpuobj_ref(struct nouveau_gpuobj *ref, struct nouveau_gpuobj **ptr)
 317 {
 318         if (ref)
 319                 kref_get(&ref->refcount);
 320
 321         if (*ptr)
 322                 kref_put(&(*ptr)->refcount, nouveau_gpuobj_del);
 323
 324         *ptr = ref;
 325 }
 326
 327 int
 328 nouveau_gpuobj_new_fake(struct drm_device *dev, u32 pinst, u64 vinst,
 329                         u32 size, u32 flags, struct nouveau_gpuobj **pgpuobj)
 330 {
 331         struct drm_nouveau_private *dev_priv = dev->dev_private;
 332         struct nouveau_gpuobj *gpuobj = NULL;
 333         int i;
 334
 335         NV_DEBUG(dev,
 336                  "pinst=0x%08x vinst=0x%010llx size=0x%08x flags=0x%08x\n",
 337                  pinst, vinst, size, flags);
 338
 339         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 340         if (!gpuobj)
 341                 return -ENOMEM;
 342         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 343         gpuobj->dev = dev;
 344         gpuobj->flags = flags;
 345         kref_init(&gpuobj->refcount);
 346         gpuobj->size  = size;
 347         gpuobj->pinst = pinst;
 348         gpuobj->cinst = NVOBJ_CINST_GLOBAL;
 349         gpuobj->vinst = vinst;
 350
 351         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 352                 for (i = 0; i < gpuobj->size; i += 4)
 353                         nv_wo32(gpuobj, i, 0);
 354                 dev_priv->engine.instmem.flush(dev);
 355         }
 356
 357         spin_lock(&dev_priv->ramin_lock);
 358         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 359         spin_unlock(&dev_priv->ramin_lock);
 360         *pgpuobj = gpuobj;
 361         return 0;
 362 }
 363
 364 /*
 365    DMA objects are used to reference a piece of memory in the
 366    framebuffer, PCI or AGP address space. Each object is 16 bytes big
 367    and looks as follows:
 368
 369    entry[0]
 370    11:0  class (seems like I can always use 0 here)
 371    12    page table present?
 372    13    page entry linear?
 373    15:14 access: 0 rw, 1 ro, 2 wo
 374    17:16 target: 0 NV memory, 1 NV memory tiled, 2 PCI, 3 AGP
 375    31:20 dma adjust (bits 0-11 of the address)
 376    entry[1]
 377    dma limit (size of transfer)
 378    entry[X]
 379    1     0 readonly, 1 readwrite
 380    31:12 dma frame address of the page (bits 12-31 of the address)
 381    entry[N]
 382    page table terminator, same value as the first pte, as does nvidia
 383    rivatv uses 0xffffffff
 384
 385    Non linear page tables need a list of frame addresses afterwards,
 386    the rivatv project has some info on this.
 387
 388    The method below creates a DMA object in instance RAM and returns a handle
 389    to it that can be used to set up context objects.
 390 */
 391
 392 void
 393 nv50_gpuobj_dma_init(struct nouveau_gpuobj *obj, u32 offset, int class,
 394                      u64 base, u64 size, int target, int access,
 395                      u32 type, u32 comp)
 396 {
 397         struct drm_nouveau_private *dev_priv = obj->dev->dev_private;
 398         struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
 399         u32 flags0;
 400
 401         flags0  = (comp << 29) | (type << 22) | class;
 402         flags0 |= 0x00100000;
 403
 404         switch (access) {
 405         case NV_MEM_ACCESS_RO: flags0 |= 0x00040000; break;
 406         case NV_MEM_ACCESS_RW:
 407         case NV_MEM_ACCESS_WO: flags0 |= 0x00080000; break;
 408         default:
 409                 break;
 410         }
 411
 412         switch (target) {
 413         case NV_MEM_TARGET_VRAM:
 414                 flags0 |= 0x00010000;
 415                 break;
 416         case NV_MEM_TARGET_PCI:
 417                 flags0 |= 0x00020000;
 418                 break;
 419         case NV_MEM_TARGET_PCI_NOSNOOP:
 420                 flags0 |= 0x00030000;
 421                 break;
 422         case NV_MEM_TARGET_GART:
 423                 base += dev_priv->gart_info.aper_base;
 424         default:
 425                 flags0 &= ~0x00100000;
 426                 break;
 427         }
 428
 429         /* convert to base + limit */
 430         size = (base + size) - 1;
 431
 432         nv_wo32(obj, offset + 0x00, flags0);
 433         nv_wo32(obj, offset + 0x04, lower_32_bits(size));
 434         nv_wo32(obj, offset + 0x08, lower_32_bits(base));
 435         nv_wo32(obj, offset + 0x0c, upper_32_bits(size) << 24 |
 436                                     upper_32_bits(base));
 437         nv_wo32(obj, offset + 0x10, 0x00000000);
 438         nv_wo32(obj, offset + 0x14, 0x00000000);
 439
 440         pinstmem->flush(obj->dev);
 441 }
 442
 443 int
 444 nv50_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base, u64 size,
 445                     int target, int access, u32 type, u32 comp,
 446                     struct nouveau_gpuobj **pobj)
 447 {
 448         struct drm_device *dev = chan->dev;
 449         int ret;
 450
 451         ret = nouveau_gpuobj_new(dev, chan, 24, 16, NVOBJ_FLAG_ZERO_FREE, pobj);
 452         if (ret)
 453                 return ret;
 454
 455         nv50_gpuobj_dma_init(*pobj, 0, class, base, size, target,
 456                              access, type, comp);
 457         return 0;
 458 }
 459
 460 int
 461 nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base,
 462                        u64 size, int access, int target,
 463                        struct nouveau_gpuobj **pobj)
 464 {
 465         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 466         struct drm_device *dev = chan->dev;
 467         struct nouveau_gpuobj *obj;
 468         u32 flags0, flags2;
 469         int ret;
 470
 471         if (dev_priv->card_type >= NV_50) {
 472                 u32 comp = (target == NV_MEM_TARGET_VM) ? NV_MEM_COMP_VM : 0;
 473                 u32 type = (target == NV_MEM_TARGET_VM) ? NV_MEM_TYPE_VM : 0;
 474
 475                 return nv50_gpuobj_dma_new(chan, class, base, size,
 476                                            target, access, type, comp, pobj);
 477         }
 478
 479         if (target == NV_MEM_TARGET_GART) {
 480                 struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
 481
 482                 if (dev_priv->gart_info.type == NOUVEAU_GART_PDMA) {
 483                         if (base == 0) {
 484                                 nouveau_gpuobj_ref(gart, pobj);
 485                                 return 0;
 486                         }
 487
 488                         base   = nouveau_sgdma_get_physical(dev, base);
 489                         target = NV_MEM_TARGET_PCI;
 490                 } else {
 491                         base += dev_priv->gart_info.aper_base;
 492                         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP)
 493                                 target = NV_MEM_TARGET_PCI_NOSNOOP;
 494                         else
 495                                 target = NV_MEM_TARGET_PCI;
 496                 }
 497         }
 498
 499         flags0  = class;
 500         flags0 |= 0x00003000; /* PT present, PT linear */
 501         flags2  = 0;
 502
 503         switch (target) {
 504         case NV_MEM_TARGET_PCI:
 505                 flags0 |= 0x00020000;
 506                 break;
 507         case NV_MEM_TARGET_PCI_NOSNOOP:
 508                 flags0 |= 0x00030000;
 509                 break;
 510         default:
 511                 break;
 512         }
 513
 514         switch (access) {
 515         case NV_MEM_ACCESS_RO:
 516                 flags0 |= 0x00004000;
 517                 break;
 518         case NV_MEM_ACCESS_WO:
 519                 flags0 |= 0x00008000;
 520         default:
 521                 flags2 |= 0x00000002;
 522                 break;
 523         }
 524
 525         flags0 |= (base & 0x00000fff) << 20;
 526         flags2 |= (base & 0xfffff000);
 527
 528         ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
 529         if (ret)
 530                 return ret;
 531
 532         nv_wo32(obj, 0x00, flags0);
 533         nv_wo32(obj, 0x04, size - 1);
 534         nv_wo32(obj, 0x08, flags2);
 535         nv_wo32(obj, 0x0c, flags2);
 536
 537         obj->engine = NVOBJ_ENGINE_SW;
 538         obj->class  = class;
 539         *pobj = obj;
 540         return 0;
 541 }
 542
 543 /* Context objects in the instance RAM have the following structure.
 544  * On NV40 they are 32 byte long, on NV30 and smaller 16 bytes.
 545
 546    NV4 - NV30:
 547
 548    entry[0]
 549    11:0 class
 550    12   chroma key enable
 551    13   user clip enable
 552    14   swizzle enable
 553    17:15 patch config:
 554        scrcopy_and, rop_and, blend_and, scrcopy, srccopy_pre, blend_pre
 555    18   synchronize enable
 556    19   endian: 1 big, 0 little
 557    21:20 dither mode
 558    23    single step enable
 559    24    patch status: 0 invalid, 1 valid
 560    25    context_surface 0: 1 valid
 561    26    context surface 1: 1 valid
 562    27    context pattern: 1 valid
 563    28    context rop: 1 valid
 564    29,30 context beta, beta4
 565    entry[1]
 566    7:0   mono format
 567    15:8  color format
 568    31:16 notify instance address
 569    entry[2]
 570    15:0  dma 0 instance address
 571    31:16 dma 1 instance address
 572    entry[3]
 573    dma method traps
 574
 575    NV40:
 576    No idea what the exact format is. Here's what can be deducted:
 577
 578    entry[0]:
 579    11:0  class  (maybe uses more bits here?)
 580    17    user clip enable
 581    21:19 patch config
 582    25    patch status valid ?
 583    entry[1]:
 584    15:0  DMA notifier  (maybe 20:0)
 585    entry[2]:
 586    15:0  DMA 0 instance (maybe 20:0)
 587    24    big endian
 588    entry[3]:
 589    15:0  DMA 1 instance (maybe 20:0)
 590    entry[4]:
 591    entry[5]:
 592    set to 0?
 593 */
 594 static int
 595 nouveau_gpuobj_sw_new(struct nouveau_channel *chan, u32 handle, u16 class)
 596 {
 597         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 598         struct nouveau_gpuobj *gpuobj;
 599         int ret;
 600
 601         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 602         if (!gpuobj)
 603                 return -ENOMEM;
 604         gpuobj->dev = chan->dev;
 605         gpuobj->engine = NVOBJ_ENGINE_SW;
 606         gpuobj->class = class;
 607         kref_init(&gpuobj->refcount);
 608         gpuobj->cinst = 0x40;
 609
 610         spin_lock(&dev_priv->ramin_lock);
 611         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 612         spin_unlock(&dev_priv->ramin_lock);
 613
 614         ret = nouveau_ramht_insert(chan, handle, gpuobj);
 615         nouveau_gpuobj_ref(NULL, &gpuobj);
 616         return ret;
 617 }
 618
 619 int
 620 nouveau_gpuobj_gr_new(struct nouveau_channel *chan, u32 handle, int class)
 621 {
 622         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 623         struct drm_device *dev = chan->dev;
 624         struct nouveau_gpuobj_class *oc;
 625         int ret;
 626
 627         NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
 628
 629         list_for_each_entry(oc, &dev_priv->classes, head) {
 630                 struct nouveau_exec_engine *eng = dev_priv->eng[oc->engine];
 631
 632                 if (oc->id != class)
 633                         continue;
 634
 635                 if (oc->engine == NVOBJ_ENGINE_SW)
 636                         return nouveau_gpuobj_sw_new(chan, handle, class);
 637
 638                 if (!chan->engctx[oc->engine]) {
 639                         ret = eng->context_new(chan, oc->engine);
 640                         if (ret)
 641                                 return ret;
 642                 }
 643
 644                 return eng->object_new(chan, oc->engine, handle, class);
 645         }
 646
 647         NV_ERROR(dev, "illegal object class: 0x%x\n", class);
 648         return -EINVAL;
 649 }
 650
 651 static int
 652 nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
 653 {
 654         struct drm_device *dev = chan->dev;
 655         struct drm_nouveau_private *dev_priv = dev->dev_private;
 656         uint32_t size;
 657         uint32_t base;
 658         int ret;
 659
 660         NV_DEBUG(dev, "ch%d\n", chan->id);
 661
 662         /* Base amount for object storage (4KiB enough?) */
 663         size = 0x2000;
 664         base = 0;
 665
 666         if (dev_priv->card_type == NV_50) {
 667                 /* Various fixed table thingos */
 668                 size += 0x1400; /* mostly unknown stuff */
 669                 size += 0x4000; /* vm pd */
 670                 base  = 0x6000;
 671                 /* RAMHT, not sure about setting size yet, 32KiB to be safe */
 672                 size += 0x8000;
 673                 /* RAMFC */
 674                 size += 0x1000;
 675         }
 676
 677         ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
 678         if (ret) {
 679                 NV_ERROR(dev, "Error allocating channel PRAMIN: %d\n", ret);
 680                 return ret;
 681         }
 682
 683         ret = drm_mm_init(&chan->ramin_heap, base, size);
 684         if (ret) {
 685                 NV_ERROR(dev, "Error creating PRAMIN heap: %d\n", ret);
 686                 nouveau_gpuobj_ref(NULL, &chan->ramin);
 687                 return ret;
 688         }
 689
 690         return 0;
 691 }
 692
 693 static int
 694 nvc0_gpuobj_channel_init(struct nouveau_channel *chan, struct nouveau_vm *vm)
 695 {
 696         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 697         struct drm_device *dev = chan->dev;
 698         struct nouveau_gpuobj *pgd = NULL;
 699         struct nouveau_vm_pgd *vpgd;
 700         int ret, i;
 701
 702         ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0, &chan->ramin);
 703         if (ret)
 704                 return ret;
 705
 706         /* create page directory for this vm if none currently exists,
 707          * will be destroyed automagically when last reference to the
 708          * vm is removed
 709          */
 710         if (list_empty(&vm->pgd_list)) {
 711                 ret = nouveau_gpuobj_new(dev, NULL, 65536, 0x1000, 0, &pgd);
 712                 if (ret)
 713                         return ret;
 714         }
 715         nouveau_vm_ref(vm, &chan->vm, pgd);
 716         nouveau_gpuobj_ref(NULL, &pgd);
 717
 718         /* point channel at vm's page directory */
 719         vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
 720         nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
 721         nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
 722         nv_wo32(chan->ramin, 0x0208, 0xffffffff);
 723         nv_wo32(chan->ramin, 0x020c, 0x000000ff);
 724
 725         /* map display semaphore buffers into channel's vm */
 726         if (dev_priv->card_type >= NV_D0)
 727                 return 0;
 728
 729         for (i = 0; i < 2; i++) {
 730                 struct nv50_display_crtc *dispc = &nv50_display(dev)->crtc[i];
 731
 732                 ret = nouveau_bo_vma_add(dispc->sem.bo, chan->vm,
 733                                          &chan->dispc_vma[i]);
 734                 if (ret)
 735                         return ret;
 736         }
 737
 738         return 0;
 739 }
 740
 741 int
 742 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 743                             uint32_t vram_h, uint32_t tt_h)
 744 {
 745         struct drm_device *dev = chan->dev;
 746         struct drm_nouveau_private *dev_priv = dev->dev_private;
 747         struct nouveau_fpriv *fpriv = nouveau_fpriv(chan->file_priv);
 748         struct nouveau_vm *vm = fpriv ? fpriv->vm : dev_priv->chan_vm;
 749         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
 750         int ret, i;
 751
 752         NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
 753         if (dev_priv->card_type >= NV_C0)
 754                 return nvc0_gpuobj_channel_init(chan, vm);
 755
 756         /* Allocate a chunk of memory for per-channel object storage */
 757         ret = nouveau_gpuobj_channel_init_pramin(chan);
 758         if (ret) {
 759                 NV_ERROR(dev, "init pramin\n");
 760                 return ret;
 761         }
 762
 763         /* NV50 VM
 764          *  - Allocate per-channel page-directory
 765          *  - Link with shared channel VM
 766          */
 767         if (vm) {
 768                 u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
 769                 u64 vm_vinst = chan->ramin->vinst + pgd_offs;
 770                 u32 vm_pinst = chan->ramin->pinst;
 771
 772                 if (vm_pinst != ~0)
 773                         vm_pinst += pgd_offs;
 774
 775                 ret = nouveau_gpuobj_new_fake(dev, vm_pinst, vm_vinst, 0x4000,
 776                                               0, &chan->vm_pd);
 777                 if (ret)
 778                         return ret;
 779
 780                 nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
 781         }
 782
 783         /* RAMHT */
 784         if (dev_priv->card_type < NV_50) {
 785                 nouveau_ramht_ref(dev_priv->ramht, &chan->ramht, NULL);
 786         } else {
 787                 struct nouveau_gpuobj *ramht = NULL;
 788
 789                 ret = nouveau_gpuobj_new(dev, chan, 0x8000, 16,
 790                                          NVOBJ_FLAG_ZERO_ALLOC, &ramht);
 791                 if (ret)
 792                         return ret;
 793
 794                 ret = nouveau_ramht_new(dev, ramht, &chan->ramht);
 795                 nouveau_gpuobj_ref(NULL, &ramht);
 796                 if (ret)
 797                         return ret;
 798
 799                 /* dma objects for display sync channel semaphore blocks */
 800                 for (i = 0; i < dev->mode_config.num_crtc; i++) {
 801                         struct nouveau_gpuobj *sem = NULL;
 802                         struct nv50_display_crtc *dispc =
 803                                 &nv50_display(dev)->crtc[i];
 804                         u64 offset = dispc->sem.bo->bo.offset;
 805
 806                         ret = nouveau_gpuobj_dma_new(chan, 0x3d, offset, 0xfff,
 807                                                      NV_MEM_ACCESS_RW,
 808                                                      NV_MEM_TARGET_VRAM, &sem);
 809                         if (ret)
 810                                 return ret;
 811
 812                         ret = nouveau_ramht_insert(chan, NvEvoSema0 + i, sem);
 813                         nouveau_gpuobj_ref(NULL, &sem);
 814                         if (ret)
 815                                 return ret;
 816                 }
 817         }
 818
 819         /* VRAM ctxdma */
 820         if (dev_priv->card_type >= NV_50) {
 821                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 822                                              0, (1ULL << 40), NV_MEM_ACCESS_RW,
 823                                              NV_MEM_TARGET_VM, &vram);
 824                 if (ret) {
 825                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 826                         return ret;
 827                 }
 828         } else {
 829                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 830                                              0, dev_priv->fb_available_size,
 831                                              NV_MEM_ACCESS_RW,
 832                                              NV_MEM_TARGET_VRAM, &vram);
 833                 if (ret) {
 834                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 835                         return ret;
 836                 }
 837         }
 838
 839         ret = nouveau_ramht_insert(chan, vram_h, vram);
 840         nouveau_gpuobj_ref(NULL, &vram);
 841         if (ret) {
 842                 NV_ERROR(dev, "Error adding VRAM ctxdma to RAMHT: %d\n", ret);
 843                 return ret;
 844         }
 845
 846         /* TT memory ctxdma */
 847         if (dev_priv->card_type >= NV_50) {
 848                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 849                                              0, (1ULL << 40), NV_MEM_ACCESS_RW,
 850                                              NV_MEM_TARGET_VM, &tt);
 851         } else {
 852                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 853                                              0, dev_priv->gart_info.aper_size,
 854                                              NV_MEM_ACCESS_RW,
 855                                              NV_MEM_TARGET_GART, &tt);
 856         }
 857
 858         if (ret) {
 859                 NV_ERROR(dev, "Error creating TT ctxdma: %d\n", ret);
 860                 return ret;
 861         }
 862
 863         ret = nouveau_ramht_insert(chan, tt_h, tt);
 864         nouveau_gpuobj_ref(NULL, &tt);
 865         if (ret) {
 866                 NV_ERROR(dev, "Error adding TT ctxdma to RAMHT: %d\n", ret);
 867                 return ret;
 868         }
 869
 870         return 0;
 871 }
 872
 873 void
 874 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
 875 {
 876         struct drm_device *dev = chan->dev;
 877         struct drm_nouveau_private *dev_priv = dev->dev_private;
 878         int i;
 879
 880         NV_DEBUG(dev, "ch%d\n", chan->id);
 881
 882         if (dev_priv->card_type >= NV_50 && dev_priv->card_type <= NV_C0) {
 883                 struct nv50_display *disp = nv50_display(dev);
 884
 885                 for (i = 0; i < dev->mode_config.num_crtc; i++) {
 886                         struct nv50_display_crtc *dispc = &disp->crtc[i];
 887                         nouveau_bo_vma_del(dispc->sem.bo, &chan->dispc_vma[i]);
 888                 }
 889         }
 890
 891         nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
 892         nouveau_gpuobj_ref(NULL, &chan->vm_pd);
 893
 894         if (drm_mm_initialized(&chan->ramin_heap))
 895                 drm_mm_takedown(&chan->ramin_heap);
 896         nouveau_gpuobj_ref(NULL, &chan->ramin);
 897 }
 898
 899 int
 900 nouveau_gpuobj_suspend(struct drm_device *dev)
 901 {
 902         struct drm_nouveau_private *dev_priv = dev->dev_private;
 903         struct nouveau_gpuobj *gpuobj;
 904         int i;
 905
 906         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 907                 if (gpuobj->cinst != NVOBJ_CINST_GLOBAL)
 908                         continue;
 909
 910                 gpuobj->suspend = vmalloc(gpuobj->size);
 911                 if (!gpuobj->suspend) {
 912                         nouveau_gpuobj_resume(dev);
 913                         return -ENOMEM;
 914                 }
 915
 916                 for (i = 0; i < gpuobj->size; i += 4)
 917                         gpuobj->suspend[i/4] = nv_ro32(gpuobj, i);
 918         }
 919
 920         return 0;
 921 }
 922
 923 void
 924 nouveau_gpuobj_resume(struct drm_device *dev)
 925 {
 926         struct drm_nouveau_private *dev_priv = dev->dev_private;
 927         struct nouveau_gpuobj *gpuobj;
 928         int i;
 929
 930         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 931                 if (!gpuobj->suspend)
 932                         continue;
 933
 934                 for (i = 0; i < gpuobj->size; i += 4)
 935                         nv_wo32(gpuobj, i, gpuobj->suspend[i/4]);
 936
 937                 vfree(gpuobj->suspend);
 938                 gpuobj->suspend = NULL;
 939         }
 940
 941         dev_priv->engine.instmem.flush(dev);
 942 }
 943
 944 int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
 945                               struct drm_file *file_priv)
 946 {
 947         struct drm_nouveau_grobj_alloc *init = data;
 948         struct nouveau_channel *chan;
 949         int ret;
 950
 951         if (init->handle == ~0)
 952                 return -EINVAL;
 953
 954         chan = nouveau_channel_get(file_priv, init->channel);
 955         if (IS_ERR(chan))
 956                 return PTR_ERR(chan);
 957
 958         if (nouveau_ramht_find(chan, init->handle)) {
 959                 ret = -EEXIST;
 960                 goto out;
 961         }
 962
 963         ret = nouveau_gpuobj_gr_new(chan, init->handle, init->class);
 964         if (ret) {
 965                 NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n",
 966                          ret, init->channel, init->handle);
 967         }
 968
 969 out:
 970         nouveau_channel_put(&chan);
 971         return ret;
 972 }
 973
 974 int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
 975                               struct drm_file *file_priv)
 976 {
 977         struct drm_nouveau_gpuobj_free *objfree = data;
 978         struct nouveau_channel *chan;
 979         int ret;
 980
 981         chan = nouveau_channel_get(file_priv, objfree->channel);
 982         if (IS_ERR(chan))
 983                 return PTR_ERR(chan);
 984
 985         /* Synchronize with the user channel */
 986         nouveau_channel_idle(chan);
 987
 988         ret = nouveau_ramht_remove(chan, objfree->handle);
 989         nouveau_channel_put(&chan);
 990         return ret;
 991 }
 992
 993 u32
 994 nv_ro32(struct nouveau_gpuobj *gpuobj, u32 offset)
 995 {
 996         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
 997         struct drm_device *dev = gpuobj->dev;
 998         unsigned long flags;
 999
1000         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
1001                 u64  ptr = gpuobj->vinst + offset;
1002                 u32 base = ptr >> 16;
1003                 u32  val;
1004
1005                 spin_lock_irqsave(&dev_priv->vm_lock, flags);
1006                 if (dev_priv->ramin_base != base) {
1007                         dev_priv->ramin_base = base;
1008                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
1009                 }
1010                 val = nv_rd32(dev, 0x700000 + (ptr & 0xffff));
1011                 spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
1012                 return val;
1013         }
1014
1015         return nv_ri32(dev, gpuobj->pinst + offset);
1016 }
1017
1018 void
1019 nv_wo32(struct nouveau_gpuobj *gpuobj, u32 offset, u32 val)
1020 {
1021         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
1022         struct drm_device *dev = gpuobj->dev;
1023         unsigned long flags;
1024
1025         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
1026                 u64  ptr = gpuobj->vinst + offset;
1027                 u32 base = ptr >> 16;
1028
1029                 spin_lock_irqsave(&dev_priv->vm_lock, flags);
1030                 if (dev_priv->ramin_base != base) {
1031                         dev_priv->ramin_base = base;
1032                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
1033                 }
1034                 nv_wr32(dev, 0x700000 + (ptr & 0xffff), val);
1035                 spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
1036                 return;
1037         }
1038
1039         nv_wi32(dev, gpuobj->pinst + offset, val);
1040 }