drivers/gpu/drm/nouveau/nouveau_object.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drv.h"
  36 #include "nouveau_drm.h"
  37 #include "nouveau_ramht.h"
  38 #include "nouveau_vm.h"
  39 #include "nv50_display.h"
  40
  41 struct nouveau_gpuobj_method {
  42         struct list_head head;
  43         u32 mthd;
  44         int (*exec)(struct nouveau_channel *, u32 class, u32 mthd, u32 data);
  45 };
  46
  47 struct nouveau_gpuobj_class {
  48         struct list_head head;
  49         struct list_head methods;
  50         u32 id;
  51         u32 engine;
  52 };
  53
  54 int
  55 nouveau_gpuobj_class_new(struct drm_device *dev, u32 class, u32 engine)
  56 {
  57         struct drm_nouveau_private *dev_priv = dev->dev_private;
  58         struct nouveau_gpuobj_class *oc;
  59
  60         oc = kzalloc(sizeof(*oc), GFP_KERNEL);
  61         if (!oc)
  62                 return -ENOMEM;
  63
  64         INIT_LIST_HEAD(&oc->methods);
  65         oc->id = class;
  66         oc->engine = engine;
  67         list_add(&oc->head, &dev_priv->classes);
  68         return 0;
  69 }
  70
  71 int
  72 nouveau_gpuobj_mthd_new(struct drm_device *dev, u32 class, u32 mthd,
  73                         int (*exec)(struct nouveau_channel *, u32, u32, u32))
  74 {
  75         struct drm_nouveau_private *dev_priv = dev->dev_private;
  76         struct nouveau_gpuobj_method *om;
  77         struct nouveau_gpuobj_class *oc;
  78
  79         list_for_each_entry(oc, &dev_priv->classes, head) {
  80                 if (oc->id == class)
  81                         goto found;
  82         }
  83
  84         return -EINVAL;
  85
  86 found:
  87         om = kzalloc(sizeof(*om), GFP_KERNEL);
  88         if (!om)
  89                 return -ENOMEM;
  90
  91         om->mthd = mthd;
  92         om->exec = exec;
  93         list_add(&om->head, &oc->methods);
  94         return 0;
  95 }
  96
  97 int
  98 nouveau_gpuobj_mthd_call(struct nouveau_channel *chan,
  99                          u32 class, u32 mthd, u32 data)
 100 {
 101         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 102         struct nouveau_gpuobj_method *om;
 103         struct nouveau_gpuobj_class *oc;
 104
 105         list_for_each_entry(oc, &dev_priv->classes, head) {
 106                 if (oc->id != class)
 107                         continue;
 108
 109                 list_for_each_entry(om, &oc->methods, head) {
 110                         if (om->mthd == mthd)
 111                                 return om->exec(chan, class, mthd, data);
 112                 }
 113         }
 114
 115         return -ENOENT;
 116 }
 117
 118 int
 119 nouveau_gpuobj_mthd_call2(struct drm_device *dev, int chid,
 120                           u32 class, u32 mthd, u32 data)
 121 {
 122         struct drm_nouveau_private *dev_priv = dev->dev_private;
 123         struct nouveau_channel *chan = NULL;
 124         unsigned long flags;
 125         int ret = -EINVAL;
 126
 127         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 128         if (chid > 0 && chid < dev_priv->engine.fifo.channels)
 129                 chan = dev_priv->channels.ptr[chid];
 130         if (chan)
 131                 ret = nouveau_gpuobj_mthd_call(chan, class, mthd, data);
 132         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 133         return ret;
 134 }
 135
 136 /* NVidia uses context objects to drive drawing operations.
 137
 138    Context objects can be selected into 8 subchannels in the FIFO,
 139    and then used via DMA command buffers.
 140
 141    A context object is referenced by a user defined handle (CARD32). The HW
 142    looks up graphics objects in a hash table in the instance RAM.
 143
 144    An entry in the hash table consists of 2 CARD32. The first CARD32 contains
 145    the handle, the second one a bitfield, that contains the address of the
 146    object in instance RAM.
 147
 148    The format of the second CARD32 seems to be:
 149
 150    NV4 to NV30:
 151
 152    15: 0  instance_addr >> 4
 153    17:16  engine (here uses 1 = graphics)
 154    28:24  channel id (here uses 0)
 155    31     valid (use 1)
 156
 157    NV40:
 158
 159    15: 0  instance_addr >> 4   (maybe 19-0)
 160    21:20  engine (here uses 1 = graphics)
 161    I'm unsure about the other bits, but using 0 seems to work.
 162
 163    The key into the hash table depends on the object handle and channel id and
 164    is given as:
 165 */
 166
 167 int
 168 nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
 169                    uint32_t size, int align, uint32_t flags,
 170                    struct nouveau_gpuobj **gpuobj_ret)
 171 {
 172         struct drm_nouveau_private *dev_priv = dev->dev_private;
 173         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 174         struct nouveau_gpuobj *gpuobj;
 175         struct drm_mm_node *ramin = NULL;
 176         int ret, i;
 177
 178         NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n",
 179                  chan ? chan->id : -1, size, align, flags);
 180
 181         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 182         if (!gpuobj)
 183                 return -ENOMEM;
 184         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 185         gpuobj->dev = dev;
 186         gpuobj->flags = flags;
 187         kref_init(&gpuobj->refcount);
 188         gpuobj->size = size;
 189
 190         spin_lock(&dev_priv->ramin_lock);
 191         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 192         spin_unlock(&dev_priv->ramin_lock);
 193
 194         if (chan) {
 195                 ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
 196                 if (ramin)
 197                         ramin = drm_mm_get_block(ramin, size, align);
 198                 if (!ramin) {
 199                         nouveau_gpuobj_ref(NULL, &gpuobj);
 200                         return -ENOMEM;
 201                 }
 202
 203                 gpuobj->pinst = chan->ramin->pinst;
 204                 if (gpuobj->pinst != ~0)
 205                         gpuobj->pinst += ramin->start;
 206
 207                 gpuobj->cinst = ramin->start;
 208                 gpuobj->vinst = ramin->start + chan->ramin->vinst;
 209                 gpuobj->node  = ramin;
 210         } else {
 211                 ret = instmem->get(gpuobj, size, align);
 212                 if (ret) {
 213                         nouveau_gpuobj_ref(NULL, &gpuobj);
 214                         return ret;
 215                 }
 216
 217                 ret = -ENOSYS;
 218                 if (!(flags & NVOBJ_FLAG_DONT_MAP))
 219                         ret = instmem->map(gpuobj);
 220                 if (ret)
 221                         gpuobj->pinst = ~0;
 222
 223                 gpuobj->cinst = NVOBJ_CINST_GLOBAL;
 224         }
 225
 226         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 227                 for (i = 0; i < gpuobj->size; i += 4)
 228                         nv_wo32(gpuobj, i, 0);
 229                 instmem->flush(dev);
 230         }
 231
 232
 233         *gpuobj_ret = gpuobj;
 234         return 0;
 235 }
 236
 237 int
 238 nouveau_gpuobj_init(struct drm_device *dev)
 239 {
 240         struct drm_nouveau_private *dev_priv = dev->dev_private;
 241
 242         NV_DEBUG(dev, "\n");
 243
 244         INIT_LIST_HEAD(&dev_priv->gpuobj_list);
 245         INIT_LIST_HEAD(&dev_priv->classes);
 246         spin_lock_init(&dev_priv->ramin_lock);
 247         dev_priv->ramin_base = ~0;
 248
 249         return 0;
 250 }
 251
 252 void
 253 nouveau_gpuobj_takedown(struct drm_device *dev)
 254 {
 255         struct drm_nouveau_private *dev_priv = dev->dev_private;
 256         struct nouveau_gpuobj_method *om, *tm;
 257         struct nouveau_gpuobj_class *oc, *tc;
 258
 259         NV_DEBUG(dev, "\n");
 260
 261         list_for_each_entry_safe(oc, tc, &dev_priv->classes, head) {
 262                 list_for_each_entry_safe(om, tm, &oc->methods, head) {
 263                         list_del(&om->head);
 264                         kfree(om);
 265                 }
 266                 list_del(&oc->head);
 267                 kfree(oc);
 268         }
 269
 270         BUG_ON(!list_empty(&dev_priv->gpuobj_list));
 271 }
 272
 273
 274 static void
 275 nouveau_gpuobj_del(struct kref *ref)
 276 {
 277         struct nouveau_gpuobj *gpuobj =
 278                 container_of(ref, struct nouveau_gpuobj, refcount);
 279         struct drm_device *dev = gpuobj->dev;
 280         struct drm_nouveau_private *dev_priv = dev->dev_private;
 281         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 282         int i;
 283
 284         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 285
 286         if (gpuobj->node && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
 287                 for (i = 0; i < gpuobj->size; i += 4)
 288                         nv_wo32(gpuobj, i, 0);
 289                 instmem->flush(dev);
 290         }
 291
 292         if (gpuobj->dtor)
 293                 gpuobj->dtor(dev, gpuobj);
 294
 295         if (gpuobj->cinst == NVOBJ_CINST_GLOBAL) {
 296                 if (gpuobj->node) {
 297                         instmem->unmap(gpuobj);
 298                         instmem->put(gpuobj);
 299                 }
 300         } else {
 301                 if (gpuobj->node) {
 302                         spin_lock(&dev_priv->ramin_lock);
 303                         drm_mm_put_block(gpuobj->node);
 304                         spin_unlock(&dev_priv->ramin_lock);
 305                 }
 306         }
 307
 308         spin_lock(&dev_priv->ramin_lock);
 309         list_del(&gpuobj->list);
 310         spin_unlock(&dev_priv->ramin_lock);
 311
 312         kfree(gpuobj);
 313 }
 314
 315 void
 316 nouveau_gpuobj_ref(struct nouveau_gpuobj *ref, struct nouveau_gpuobj **ptr)
 317 {
 318         if (ref)
 319                 kref_get(&ref->refcount);
 320
 321         if (*ptr)
 322                 kref_put(&(*ptr)->refcount, nouveau_gpuobj_del);
 323
 324         *ptr = ref;
 325 }
 326
 327 int
 328 nouveau_gpuobj_new_fake(struct drm_device *dev, u32 pinst, u64 vinst,
 329                         u32 size, u32 flags, struct nouveau_gpuobj **pgpuobj)
 330 {
 331         struct drm_nouveau_private *dev_priv = dev->dev_private;
 332         struct nouveau_gpuobj *gpuobj = NULL;
 333         int i;
 334
 335         NV_DEBUG(dev,
 336                  "pinst=0x%08x vinst=0x%010llx size=0x%08x flags=0x%08x\n",
 337                  pinst, vinst, size, flags);
 338
 339         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 340         if (!gpuobj)
 341                 return -ENOMEM;
 342         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 343         gpuobj->dev = dev;
 344         gpuobj->flags = flags;
 345         kref_init(&gpuobj->refcount);
 346         gpuobj->size  = size;
 347         gpuobj->pinst = pinst;
 348         gpuobj->cinst = NVOBJ_CINST_GLOBAL;
 349         gpuobj->vinst = vinst;
 350
 351         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 352                 for (i = 0; i < gpuobj->size; i += 4)
 353                         nv_wo32(gpuobj, i, 0);
 354                 dev_priv->engine.instmem.flush(dev);
 355         }
 356
 357         spin_lock(&dev_priv->ramin_lock);
 358         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 359         spin_unlock(&dev_priv->ramin_lock);
 360         *pgpuobj = gpuobj;
 361         return 0;
 362 }
 363
 364 /*
 365    DMA objects are used to reference a piece of memory in the
 366    framebuffer, PCI or AGP address space. Each object is 16 bytes big
 367    and looks as follows:
 368
 369    entry[0]
 370    11:0  class (seems like I can always use 0 here)
 371    12    page table present?
 372    13    page entry linear?
 373    15:14 access: 0 rw, 1 ro, 2 wo
 374    17:16 target: 0 NV memory, 1 NV memory tiled, 2 PCI, 3 AGP
 375    31:20 dma adjust (bits 0-11 of the address)
 376    entry[1]
 377    dma limit (size of transfer)
 378    entry[X]
 379    1     0 readonly, 1 readwrite
 380    31:12 dma frame address of the page (bits 12-31 of the address)
 381    entry[N]
 382    page table terminator, same value as the first pte, as does nvidia
 383    rivatv uses 0xffffffff
 384
 385    Non linear page tables need a list of frame addresses afterwards,
 386    the rivatv project has some info on this.
 387
 388    The method below creates a DMA object in instance RAM and returns a handle
 389    to it that can be used to set up context objects.
 390 */
 391
 392 void
 393 nv50_gpuobj_dma_init(struct nouveau_gpuobj *obj, u32 offset, int class,
 394                      u64 base, u64 size, int target, int access,
 395                      u32 type, u32 comp)
 396 {
 397         struct drm_nouveau_private *dev_priv = obj->dev->dev_private;
 398         struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
 399         u32 flags0;
 400
 401         flags0  = (comp << 29) | (type << 22) | class;
 402         flags0 |= 0x00100000;
 403
 404         switch (access) {
 405         case NV_MEM_ACCESS_RO: flags0 |= 0x00040000; break;
 406         case NV_MEM_ACCESS_RW:
 407         case NV_MEM_ACCESS_WO: flags0 |= 0x00080000; break;
 408         default:
 409                 break;
 410         }
 411
 412         switch (target) {
 413         case NV_MEM_TARGET_VRAM:
 414                 flags0 |= 0x00010000;
 415                 break;
 416         case NV_MEM_TARGET_PCI:
 417                 flags0 |= 0x00020000;
 418                 break;
 419         case NV_MEM_TARGET_PCI_NOSNOOP:
 420                 flags0 |= 0x00030000;
 421                 break;
 422         case NV_MEM_TARGET_GART:
 423                 base += dev_priv->gart_info.aper_base;
 424         default:
 425                 flags0 &= ~0x00100000;
 426                 break;
 427         }
 428
 429         /* convert to base + limit */
 430         size = (base + size) - 1;
 431
 432         nv_wo32(obj, offset + 0x00, flags0);
 433         nv_wo32(obj, offset + 0x04, lower_32_bits(size));
 434         nv_wo32(obj, offset + 0x08, lower_32_bits(base));
 435         nv_wo32(obj, offset + 0x0c, upper_32_bits(size) << 24 |
 436                                     upper_32_bits(base));
 437         nv_wo32(obj, offset + 0x10, 0x00000000);
 438         nv_wo32(obj, offset + 0x14, 0x00000000);
 439
 440         pinstmem->flush(obj->dev);
 441 }
 442
 443 int
 444 nv50_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base, u64 size,
 445                     int target, int access, u32 type, u32 comp,
 446                     struct nouveau_gpuobj **pobj)
 447 {
 448         struct drm_device *dev = chan->dev;
 449         int ret;
 450
 451         ret = nouveau_gpuobj_new(dev, chan, 24, 16, NVOBJ_FLAG_ZERO_FREE, pobj);
 452         if (ret)
 453                 return ret;
 454
 455         nv50_gpuobj_dma_init(*pobj, 0, class, base, size, target,
 456                              access, type, comp);
 457         return 0;
 458 }
 459
 460 int
 461 nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base,
 462                        u64 size, int access, int target,
 463                        struct nouveau_gpuobj **pobj)
 464 {
 465         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 466         struct drm_device *dev = chan->dev;
 467         struct nouveau_gpuobj *obj;
 468         u32 flags0, flags2;
 469         int ret;
 470
 471         if (dev_priv->card_type >= NV_50) {
 472                 u32 comp = (target == NV_MEM_TARGET_VM) ? NV_MEM_COMP_VM : 0;
 473                 u32 type = (target == NV_MEM_TARGET_VM) ? NV_MEM_TYPE_VM : 0;
 474
 475                 return nv50_gpuobj_dma_new(chan, class, base, size,
 476                                            target, access, type, comp, pobj);
 477         }
 478
 479         if (target == NV_MEM_TARGET_GART) {
 480                 struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
 481
 482                 if (dev_priv->gart_info.type == NOUVEAU_GART_PDMA) {
 483                         if (base == 0) {
 484                                 nouveau_gpuobj_ref(gart, pobj);
 485                                 return 0;
 486                         }
 487
 488                         base   = nouveau_sgdma_get_physical(dev, base);
 489                         target = NV_MEM_TARGET_PCI;
 490                 } else {
 491                         base += dev_priv->gart_info.aper_base;
 492                         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP)
 493                                 target = NV_MEM_TARGET_PCI_NOSNOOP;
 494                         else
 495                                 target = NV_MEM_TARGET_PCI;
 496                 }
 497         }
 498
 499         flags0  = class;
 500         flags0 |= 0x00003000; /* PT present, PT linear */
 501         flags2  = 0;
 502
 503         switch (target) {
 504         case NV_MEM_TARGET_PCI:
 505                 flags0 |= 0x00020000;
 506                 break;
 507         case NV_MEM_TARGET_PCI_NOSNOOP:
 508                 flags0 |= 0x00030000;
 509                 break;
 510         default:
 511                 break;
 512         }
 513
 514         switch (access) {
 515         case NV_MEM_ACCESS_RO:
 516                 flags0 |= 0x00004000;
 517                 break;
 518         case NV_MEM_ACCESS_WO:
 519                 flags0 |= 0x00008000;
 520         default:
 521                 flags2 |= 0x00000002;
 522                 break;
 523         }
 524
 525         flags0 |= (base & 0x00000fff) << 20;
 526         flags2 |= (base & 0xfffff000);
 527
 528         ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
 529         if (ret)
 530                 return ret;
 531
 532         nv_wo32(obj, 0x00, flags0);
 533         nv_wo32(obj, 0x04, size - 1);
 534         nv_wo32(obj, 0x08, flags2);
 535         nv_wo32(obj, 0x0c, flags2);
 536
 537         obj->engine = NVOBJ_ENGINE_SW;
 538         obj->class  = class;
 539         *pobj = obj;
 540         return 0;
 541 }
 542
 543 /* Context objects in the instance RAM have the following structure.
 544  * On NV40 they are 32 byte long, on NV30 and smaller 16 bytes.
 545
 546    NV4 - NV30:
 547
 548    entry[0]
 549    11:0 class
 550    12   chroma key enable
 551    13   user clip enable
 552    14   swizzle enable
 553    17:15 patch config:
 554        scrcopy_and, rop_and, blend_and, scrcopy, srccopy_pre, blend_pre
 555    18   synchronize enable
 556    19   endian: 1 big, 0 little
 557    21:20 dither mode
 558    23    single step enable
 559    24    patch status: 0 invalid, 1 valid
 560    25    context_surface 0: 1 valid
 561    26    context surface 1: 1 valid
 562    27    context pattern: 1 valid
 563    28    context rop: 1 valid
 564    29,30 context beta, beta4
 565    entry[1]
 566    7:0   mono format
 567    15:8  color format
 568    31:16 notify instance address
 569    entry[2]
 570    15:0  dma 0 instance address
 571    31:16 dma 1 instance address
 572    entry[3]
 573    dma method traps
 574
 575    NV40:
 576    No idea what the exact format is. Here's what can be deducted:
 577
 578    entry[0]:
 579    11:0  class  (maybe uses more bits here?)
 580    17    user clip enable
 581    21:19 patch config
 582    25    patch status valid ?
 583    entry[1]:
 584    15:0  DMA notifier  (maybe 20:0)
 585    entry[2]:
 586    15:0  DMA 0 instance (maybe 20:0)
 587    24    big endian
 588    entry[3]:
 589    15:0  DMA 1 instance (maybe 20:0)
 590    entry[4]:
 591    entry[5]:
 592    set to 0?
 593 */
 594 static int
 595 nouveau_gpuobj_sw_new(struct nouveau_channel *chan, u32 handle, u16 class)
 596 {
 597         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 598         struct nouveau_gpuobj *gpuobj;
 599         int ret;
 600
 601         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 602         if (!gpuobj)
 603                 return -ENOMEM;
 604         gpuobj->dev = chan->dev;
 605         gpuobj->engine = NVOBJ_ENGINE_SW;
 606         gpuobj->class = class;
 607         kref_init(&gpuobj->refcount);
 608         gpuobj->cinst = 0x40;
 609
 610         spin_lock(&dev_priv->ramin_lock);
 611         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 612         spin_unlock(&dev_priv->ramin_lock);
 613
 614         ret = nouveau_ramht_insert(chan, handle, gpuobj);
 615         nouveau_gpuobj_ref(NULL, &gpuobj);
 616         return ret;
 617 }
 618
 619 int
 620 nouveau_gpuobj_gr_new(struct nouveau_channel *chan, u32 handle, int class)
 621 {
 622         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 623         struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
 624         struct drm_device *dev = chan->dev;
 625         struct nouveau_gpuobj_class *oc;
 626         int ret;
 627
 628         NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
 629
 630         list_for_each_entry(oc, &dev_priv->classes, head) {
 631                 if (oc->id == class)
 632                         goto found;
 633         }
 634
 635         NV_ERROR(dev, "illegal object class: 0x%x\n", class);
 636         return -EINVAL;
 637
 638 found:
 639         if (!dev_priv->eng[oc->engine]) {
 640                 switch (oc->engine) {
 641                 case NVOBJ_ENGINE_SW:
 642                         return nouveau_gpuobj_sw_new(chan, handle, class);
 643                 case NVOBJ_ENGINE_GR:
 644                         if ((dev_priv->card_type >= NV_20 && !chan->ramin_grctx) ||
 645                             (dev_priv->card_type  < NV_20 && !chan->pgraph_ctx)) {
 646                                 ret = pgraph->create_context(chan);
 647                                 if (ret)
 648                                         return ret;
 649                         }
 650
 651                         return pgraph->object_new(chan, handle, class);
 652                 }
 653         }
 654
 655         if (!chan->engctx[oc->engine]) {
 656                 ret = dev_priv->eng[oc->engine]->context_new(chan, oc->engine);
 657                 if (ret)
 658                         return ret;
 659         }
 660
 661         return dev_priv->eng[oc->engine]->object_new(chan, oc->engine, handle, class);
 662 }
 663
 664 static int
 665 nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
 666 {
 667         struct drm_device *dev = chan->dev;
 668         struct drm_nouveau_private *dev_priv = dev->dev_private;
 669         uint32_t size;
 670         uint32_t base;
 671         int ret;
 672
 673         NV_DEBUG(dev, "ch%d\n", chan->id);
 674
 675         /* Base amount for object storage (4KiB enough?) */
 676         size = 0x2000;
 677         base = 0;
 678
 679         /* PGRAPH context */
 680         size += dev_priv->engine.graph.grctx_size;
 681
 682         if (dev_priv->card_type == NV_50) {
 683                 /* Various fixed table thingos */
 684                 size += 0x1400; /* mostly unknown stuff */
 685                 size += 0x4000; /* vm pd */
 686                 base  = 0x6000;
 687                 /* RAMHT, not sure about setting size yet, 32KiB to be safe */
 688                 size += 0x8000;
 689                 /* RAMFC */
 690                 size += 0x1000;
 691         }
 692
 693         ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
 694         if (ret) {
 695                 NV_ERROR(dev, "Error allocating channel PRAMIN: %d\n", ret);
 696                 return ret;
 697         }
 698
 699         ret = drm_mm_init(&chan->ramin_heap, base, size);
 700         if (ret) {
 701                 NV_ERROR(dev, "Error creating PRAMIN heap: %d\n", ret);
 702                 nouveau_gpuobj_ref(NULL, &chan->ramin);
 703                 return ret;
 704         }
 705
 706         return 0;
 707 }
 708
 709 int
 710 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 711                             uint32_t vram_h, uint32_t tt_h)
 712 {
 713         struct drm_device *dev = chan->dev;
 714         struct drm_nouveau_private *dev_priv = dev->dev_private;
 715         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
 716         int ret, i;
 717
 718         NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
 719
 720         if (dev_priv->card_type == NV_C0) {
 721                 struct nouveau_vm *vm = dev_priv->chan_vm;
 722                 struct nouveau_vm_pgd *vpgd;
 723
 724                 ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0,
 725                                          &chan->ramin);
 726                 if (ret)
 727                         return ret;
 728
 729                 nouveau_vm_ref(vm, &chan->vm, NULL);
 730
 731                 vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
 732                 nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
 733                 nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
 734                 nv_wo32(chan->ramin, 0x0208, 0xffffffff);
 735                 nv_wo32(chan->ramin, 0x020c, 0x000000ff);
 736                 return 0;
 737         }
 738
 739         /* Allocate a chunk of memory for per-channel object storage */
 740         ret = nouveau_gpuobj_channel_init_pramin(chan);
 741         if (ret) {
 742                 NV_ERROR(dev, "init pramin\n");
 743                 return ret;
 744         }
 745
 746         /* NV50 VM
 747          *  - Allocate per-channel page-directory
 748          *  - Link with shared channel VM
 749          */
 750         if (dev_priv->chan_vm) {
 751                 u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
 752                 u64 vm_vinst = chan->ramin->vinst + pgd_offs;
 753                 u32 vm_pinst = chan->ramin->pinst;
 754
 755                 if (vm_pinst != ~0)
 756                         vm_pinst += pgd_offs;
 757
 758                 ret = nouveau_gpuobj_new_fake(dev, vm_pinst, vm_vinst, 0x4000,
 759                                               0, &chan->vm_pd);
 760                 if (ret)
 761                         return ret;
 762
 763                 nouveau_vm_ref(dev_priv->chan_vm, &chan->vm, chan->vm_pd);
 764         }
 765
 766         /* RAMHT */
 767         if (dev_priv->card_type < NV_50) {
 768                 nouveau_ramht_ref(dev_priv->ramht, &chan->ramht, NULL);
 769         } else {
 770                 struct nouveau_gpuobj *ramht = NULL;
 771
 772                 ret = nouveau_gpuobj_new(dev, chan, 0x8000, 16,
 773                                          NVOBJ_FLAG_ZERO_ALLOC, &ramht);
 774                 if (ret)
 775                         return ret;
 776
 777                 ret = nouveau_ramht_new(dev, ramht, &chan->ramht);
 778                 nouveau_gpuobj_ref(NULL, &ramht);
 779                 if (ret)
 780                         return ret;
 781
 782                 /* dma objects for display sync channel semaphore blocks */
 783                 for (i = 0; i < 2; i++) {
 784                         struct nouveau_gpuobj *sem = NULL;
 785                         struct nv50_display_crtc *dispc =
 786                                 &nv50_display(dev)->crtc[i];
 787                         u64 offset = dispc->sem.bo->bo.mem.start << PAGE_SHIFT;
 788
 789                         ret = nouveau_gpuobj_dma_new(chan, 0x3d, offset, 0xfff,
 790                                                      NV_MEM_ACCESS_RW,
 791                                                      NV_MEM_TARGET_VRAM, &sem);
 792                         if (ret)
 793                                 return ret;
 794
 795                         ret = nouveau_ramht_insert(chan, NvEvoSema0 + i, sem);
 796                         nouveau_gpuobj_ref(NULL, &sem);
 797                         if (ret)
 798                                 return ret;
 799                 }
 800         }
 801
 802         /* VRAM ctxdma */
 803         if (dev_priv->card_type >= NV_50) {
 804                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 805                                              0, (1ULL << 40), NV_MEM_ACCESS_RW,
 806                                              NV_MEM_TARGET_VM, &vram);
 807                 if (ret) {
 808                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 809                         return ret;
 810                 }
 811         } else {
 812                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 813                                              0, dev_priv->fb_available_size,
 814                                              NV_MEM_ACCESS_RW,
 815                                              NV_MEM_TARGET_VRAM, &vram);
 816                 if (ret) {
 817                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 818                         return ret;
 819                 }
 820         }
 821
 822         ret = nouveau_ramht_insert(chan, vram_h, vram);
 823         nouveau_gpuobj_ref(NULL, &vram);
 824         if (ret) {
 825                 NV_ERROR(dev, "Error adding VRAM ctxdma to RAMHT: %d\n", ret);
 826                 return ret;
 827         }
 828
 829         /* TT memory ctxdma */
 830         if (dev_priv->card_type >= NV_50) {
 831                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 832                                              0, (1ULL << 40), NV_MEM_ACCESS_RW,
 833                                              NV_MEM_TARGET_VM, &tt);
 834         } else {
 835                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 836                                              0, dev_priv->gart_info.aper_size,
 837                                              NV_MEM_ACCESS_RW,
 838                                              NV_MEM_TARGET_GART, &tt);
 839         }
 840
 841         if (ret) {
 842                 NV_ERROR(dev, "Error creating TT ctxdma: %d\n", ret);
 843                 return ret;
 844         }
 845
 846         ret = nouveau_ramht_insert(chan, tt_h, tt);
 847         nouveau_gpuobj_ref(NULL, &tt);
 848         if (ret) {
 849                 NV_ERROR(dev, "Error adding TT ctxdma to RAMHT: %d\n", ret);
 850                 return ret;
 851         }
 852
 853         return 0;
 854 }
 855
 856 void
 857 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
 858 {
 859         struct drm_device *dev = chan->dev;
 860
 861         NV_DEBUG(dev, "ch%d\n", chan->id);
 862
 863         nouveau_ramht_ref(NULL, &chan->ramht, chan);
 864
 865         nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
 866         nouveau_gpuobj_ref(NULL, &chan->vm_pd);
 867
 868         if (drm_mm_initialized(&chan->ramin_heap))
 869                 drm_mm_takedown(&chan->ramin_heap);
 870         nouveau_gpuobj_ref(NULL, &chan->ramin);
 871 }
 872
 873 int
 874 nouveau_gpuobj_suspend(struct drm_device *dev)
 875 {
 876         struct drm_nouveau_private *dev_priv = dev->dev_private;
 877         struct nouveau_gpuobj *gpuobj;
 878         int i;
 879
 880         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 881                 if (gpuobj->cinst != NVOBJ_CINST_GLOBAL)
 882                         continue;
 883
 884                 gpuobj->suspend = vmalloc(gpuobj->size);
 885                 if (!gpuobj->suspend) {
 886                         nouveau_gpuobj_resume(dev);
 887                         return -ENOMEM;
 888                 }
 889
 890                 for (i = 0; i < gpuobj->size; i += 4)
 891                         gpuobj->suspend[i/4] = nv_ro32(gpuobj, i);
 892         }
 893
 894         return 0;
 895 }
 896
 897 void
 898 nouveau_gpuobj_resume(struct drm_device *dev)
 899 {
 900         struct drm_nouveau_private *dev_priv = dev->dev_private;
 901         struct nouveau_gpuobj *gpuobj;
 902         int i;
 903
 904         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 905                 if (!gpuobj->suspend)
 906                         continue;
 907
 908                 for (i = 0; i < gpuobj->size; i += 4)
 909                         nv_wo32(gpuobj, i, gpuobj->suspend[i/4]);
 910
 911                 vfree(gpuobj->suspend);
 912                 gpuobj->suspend = NULL;
 913         }
 914
 915         dev_priv->engine.instmem.flush(dev);
 916 }
 917
 918 int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
 919                               struct drm_file *file_priv)
 920 {
 921         struct drm_nouveau_grobj_alloc *init = data;
 922         struct nouveau_channel *chan;
 923         int ret;
 924
 925         if (init->handle == ~0)
 926                 return -EINVAL;
 927
 928         chan = nouveau_channel_get(dev, file_priv, init->channel);
 929         if (IS_ERR(chan))
 930                 return PTR_ERR(chan);
 931
 932         if (nouveau_ramht_find(chan, init->handle)) {
 933                 ret = -EEXIST;
 934                 goto out;
 935         }
 936
 937         ret = nouveau_gpuobj_gr_new(chan, init->handle, init->class);
 938         if (ret) {
 939                 NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n",
 940                          ret, init->channel, init->handle);
 941         }
 942
 943 out:
 944         nouveau_channel_put(&chan);
 945         return ret;
 946 }
 947
 948 int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
 949                               struct drm_file *file_priv)
 950 {
 951         struct drm_nouveau_gpuobj_free *objfree = data;
 952         struct nouveau_channel *chan;
 953         int ret;
 954
 955         chan = nouveau_channel_get(dev, file_priv, objfree->channel);
 956         if (IS_ERR(chan))
 957                 return PTR_ERR(chan);
 958
 959         /* Synchronize with the user channel */
 960         nouveau_channel_idle(chan);
 961
 962         ret = nouveau_ramht_remove(chan, objfree->handle);
 963         nouveau_channel_put(&chan);
 964         return ret;
 965 }
 966
 967 u32
 968 nv_ro32(struct nouveau_gpuobj *gpuobj, u32 offset)
 969 {
 970         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
 971         struct drm_device *dev = gpuobj->dev;
 972         unsigned long flags;
 973
 974         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
 975                 u64  ptr = gpuobj->vinst + offset;
 976                 u32 base = ptr >> 16;
 977                 u32  val;
 978
 979                 spin_lock_irqsave(&dev_priv->vm_lock, flags);
 980                 if (dev_priv->ramin_base != base) {
 981                         dev_priv->ramin_base = base;
 982                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
 983                 }
 984                 val = nv_rd32(dev, 0x700000 + (ptr & 0xffff));
 985                 spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
 986                 return val;
 987         }
 988
 989         return nv_ri32(dev, gpuobj->pinst + offset);
 990 }
 991
 992 void
 993 nv_wo32(struct nouveau_gpuobj *gpuobj, u32 offset, u32 val)
 994 {
 995         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
 996         struct drm_device *dev = gpuobj->dev;
 997         unsigned long flags;
 998
 999         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
1000                 u64  ptr = gpuobj->vinst + offset;
1001                 u32 base = ptr >> 16;
1002
1003                 spin_lock_irqsave(&dev_priv->vm_lock, flags);
1004                 if (dev_priv->ramin_base != base) {
1005                         dev_priv->ramin_base = base;
1006                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
1007                 }
1008                 nv_wr32(dev, 0x700000 + (ptr & 0xffff), val);
1009                 spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
1010                 return;
1011         }
1012
1013         nv_wi32(dev, gpuobj->pinst + offset, val);
1014 }