drivers/gpu/drm/nouveau/nouveau_object.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drv.h"
  36 #include "nouveau_drm.h"
  37 #include "nouveau_ramht.h"
  38 #include "nouveau_vm.h"
  39 #include "nv50_display.h"
  40
  41 struct nouveau_gpuobj_method {
  42         struct list_head head;
  43         u32 mthd;
  44         int (*exec)(struct nouveau_channel *, u32 class, u32 mthd, u32 data);
  45 };
  46
  47 struct nouveau_gpuobj_class {
  48         struct list_head head;
  49         struct list_head methods;
  50         u32 id;
  51         u32 engine;
  52 };
  53
  54 int
  55 nouveau_gpuobj_class_new(struct drm_device *dev, u32 class, u32 engine)
  56 {
  57         struct drm_nouveau_private *dev_priv = dev->dev_private;
  58         struct nouveau_gpuobj_class *oc;
  59
  60         oc = kzalloc(sizeof(*oc), GFP_KERNEL);
  61         if (!oc)
  62                 return -ENOMEM;
  63
  64         INIT_LIST_HEAD(&oc->methods);
  65         oc->id = class;
  66         oc->engine = engine;
  67         list_add(&oc->head, &dev_priv->classes);
  68         return 0;
  69 }
  70
  71 int
  72 nouveau_gpuobj_mthd_new(struct drm_device *dev, u32 class, u32 mthd,
  73                         int (*exec)(struct nouveau_channel *, u32, u32, u32))
  74 {
  75         struct drm_nouveau_private *dev_priv = dev->dev_private;
  76         struct nouveau_gpuobj_method *om;
  77         struct nouveau_gpuobj_class *oc;
  78
  79         list_for_each_entry(oc, &dev_priv->classes, head) {
  80                 if (oc->id == class)
  81                         goto found;
  82         }
  83
  84         return -EINVAL;
  85
  86 found:
  87         om = kzalloc(sizeof(*om), GFP_KERNEL);
  88         if (!om)
  89                 return -ENOMEM;
  90
  91         om->mthd = mthd;
  92         om->exec = exec;
  93         list_add(&om->head, &oc->methods);
  94         return 0;
  95 }
  96
  97 int
  98 nouveau_gpuobj_mthd_call(struct nouveau_channel *chan,
  99                          u32 class, u32 mthd, u32 data)
 100 {
 101         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 102         struct nouveau_gpuobj_method *om;
 103         struct nouveau_gpuobj_class *oc;
 104
 105         list_for_each_entry(oc, &dev_priv->classes, head) {
 106                 if (oc->id != class)
 107                         continue;
 108
 109                 list_for_each_entry(om, &oc->methods, head) {
 110                         if (om->mthd == mthd)
 111                                 return om->exec(chan, class, mthd, data);
 112                 }
 113         }
 114
 115         return -ENOENT;
 116 }
 117
 118 int
 119 nouveau_gpuobj_mthd_call2(struct drm_device *dev, int chid,
 120                           u32 class, u32 mthd, u32 data)
 121 {
 122         struct drm_nouveau_private *dev_priv = dev->dev_private;
 123         struct nouveau_channel *chan = NULL;
 124         unsigned long flags;
 125         int ret = -EINVAL;
 126
 127         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 128         if (chid > 0 && chid < dev_priv->engine.fifo.channels)
 129                 chan = dev_priv->channels.ptr[chid];
 130         if (chan)
 131                 ret = nouveau_gpuobj_mthd_call(chan, class, mthd, data);
 132         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 133         return ret;
 134 }
 135
 136 /* NVidia uses context objects to drive drawing operations.
 137
 138    Context objects can be selected into 8 subchannels in the FIFO,
 139    and then used via DMA command buffers.
 140
 141    A context object is referenced by a user defined handle (CARD32). The HW
 142    looks up graphics objects in a hash table in the instance RAM.
 143
 144    An entry in the hash table consists of 2 CARD32. The first CARD32 contains
 145    the handle, the second one a bitfield, that contains the address of the
 146    object in instance RAM.
 147
 148    The format of the second CARD32 seems to be:
 149
 150    NV4 to NV30:
 151
 152    15: 0  instance_addr >> 4
 153    17:16  engine (here uses 1 = graphics)
 154    28:24  channel id (here uses 0)
 155    31     valid (use 1)
 156
 157    NV40:
 158
 159    15: 0  instance_addr >> 4   (maybe 19-0)
 160    21:20  engine (here uses 1 = graphics)
 161    I'm unsure about the other bits, but using 0 seems to work.
 162
 163    The key into the hash table depends on the object handle and channel id and
 164    is given as:
 165 */
 166
 167 int
 168 nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
 169                    uint32_t size, int align, uint32_t flags,
 170                    struct nouveau_gpuobj **gpuobj_ret)
 171 {
 172         struct drm_nouveau_private *dev_priv = dev->dev_private;
 173         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 174         struct nouveau_gpuobj *gpuobj;
 175         struct drm_mm_node *ramin = NULL;
 176         int ret, i;
 177
 178         NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n",
 179                  chan ? chan->id : -1, size, align, flags);
 180
 181         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 182         if (!gpuobj)
 183                 return -ENOMEM;
 184         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 185         gpuobj->dev = dev;
 186         gpuobj->flags = flags;
 187         kref_init(&gpuobj->refcount);
 188         gpuobj->size = size;
 189
 190         spin_lock(&dev_priv->ramin_lock);
 191         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 192         spin_unlock(&dev_priv->ramin_lock);
 193
 194         if (chan) {
 195                 ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
 196                 if (ramin)
 197                         ramin = drm_mm_get_block(ramin, size, align);
 198                 if (!ramin) {
 199                         nouveau_gpuobj_ref(NULL, &gpuobj);
 200                         return -ENOMEM;
 201                 }
 202
 203                 gpuobj->pinst = chan->ramin->pinst;
 204                 if (gpuobj->pinst != ~0)
 205                         gpuobj->pinst += ramin->start;
 206
 207                 gpuobj->cinst = ramin->start;
 208                 gpuobj->vinst = ramin->start + chan->ramin->vinst;
 209                 gpuobj->node  = ramin;
 210         } else {
 211                 ret = instmem->get(gpuobj, size, align);
 212                 if (ret) {
 213                         nouveau_gpuobj_ref(NULL, &gpuobj);
 214                         return ret;
 215                 }
 216
 217                 ret = -ENOSYS;
 218                 if (!(flags & NVOBJ_FLAG_DONT_MAP))
 219                         ret = instmem->map(gpuobj);
 220                 if (ret)
 221                         gpuobj->pinst = ~0;
 222
 223                 gpuobj->cinst = NVOBJ_CINST_GLOBAL;
 224         }
 225
 226         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 227                 for (i = 0; i < gpuobj->size; i += 4)
 228                         nv_wo32(gpuobj, i, 0);
 229                 instmem->flush(dev);
 230         }
 231
 232
 233         *gpuobj_ret = gpuobj;
 234         return 0;
 235 }
 236
 237 int
 238 nouveau_gpuobj_init(struct drm_device *dev)
 239 {
 240         struct drm_nouveau_private *dev_priv = dev->dev_private;
 241
 242         NV_DEBUG(dev, "\n");
 243
 244         INIT_LIST_HEAD(&dev_priv->gpuobj_list);
 245         INIT_LIST_HEAD(&dev_priv->classes);
 246         spin_lock_init(&dev_priv->ramin_lock);
 247         dev_priv->ramin_base = ~0;
 248
 249         return 0;
 250 }
 251
 252 void
 253 nouveau_gpuobj_takedown(struct drm_device *dev)
 254 {
 255         struct drm_nouveau_private *dev_priv = dev->dev_private;
 256         struct nouveau_gpuobj_method *om, *tm;
 257         struct nouveau_gpuobj_class *oc, *tc;
 258
 259         NV_DEBUG(dev, "\n");
 260
 261         list_for_each_entry_safe(oc, tc, &dev_priv->classes, head) {
 262                 list_for_each_entry_safe(om, tm, &oc->methods, head) {
 263                         list_del(&om->head);
 264                         kfree(om);
 265                 }
 266                 list_del(&oc->head);
 267                 kfree(oc);
 268         }
 269
 270         BUG_ON(!list_empty(&dev_priv->gpuobj_list));
 271 }
 272
 273
 274 static void
 275 nouveau_gpuobj_del(struct kref *ref)
 276 {
 277         struct nouveau_gpuobj *gpuobj =
 278                 container_of(ref, struct nouveau_gpuobj, refcount);
 279         struct drm_device *dev = gpuobj->dev;
 280         struct drm_nouveau_private *dev_priv = dev->dev_private;
 281         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 282         int i;
 283
 284         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 285
 286         if (gpuobj->node && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
 287                 for (i = 0; i < gpuobj->size; i += 4)
 288                         nv_wo32(gpuobj, i, 0);
 289                 instmem->flush(dev);
 290         }
 291
 292         if (gpuobj->dtor)
 293                 gpuobj->dtor(dev, gpuobj);
 294
 295         if (gpuobj->cinst == NVOBJ_CINST_GLOBAL) {
 296                 if (gpuobj->node) {
 297                         instmem->unmap(gpuobj);
 298                         instmem->put(gpuobj);
 299                 }
 300         } else {
 301                 if (gpuobj->node) {
 302                         spin_lock(&dev_priv->ramin_lock);
 303                         drm_mm_put_block(gpuobj->node);
 304                         spin_unlock(&dev_priv->ramin_lock);
 305                 }
 306         }
 307
 308         spin_lock(&dev_priv->ramin_lock);
 309         list_del(&gpuobj->list);
 310         spin_unlock(&dev_priv->ramin_lock);
 311
 312         kfree(gpuobj);
 313 }
 314
 315 void
 316 nouveau_gpuobj_ref(struct nouveau_gpuobj *ref, struct nouveau_gpuobj **ptr)
 317 {
 318         if (ref)
 319                 kref_get(&ref->refcount);
 320
 321         if (*ptr)
 322                 kref_put(&(*ptr)->refcount, nouveau_gpuobj_del);
 323
 324         *ptr = ref;
 325 }
 326
 327 int
 328 nouveau_gpuobj_new_fake(struct drm_device *dev, u32 pinst, u64 vinst,
 329                         u32 size, u32 flags, struct nouveau_gpuobj **pgpuobj)
 330 {
 331         struct drm_nouveau_private *dev_priv = dev->dev_private;
 332         struct nouveau_gpuobj *gpuobj = NULL;
 333         int i;
 334
 335         NV_DEBUG(dev,
 336                  "pinst=0x%08x vinst=0x%010llx size=0x%08x flags=0x%08x\n",
 337                  pinst, vinst, size, flags);
 338
 339         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 340         if (!gpuobj)
 341                 return -ENOMEM;
 342         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 343         gpuobj->dev = dev;
 344         gpuobj->flags = flags;
 345         kref_init(&gpuobj->refcount);
 346         gpuobj->size  = size;
 347         gpuobj->pinst = pinst;
 348         gpuobj->cinst = NVOBJ_CINST_GLOBAL;
 349         gpuobj->vinst = vinst;
 350
 351         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 352                 for (i = 0; i < gpuobj->size; i += 4)
 353                         nv_wo32(gpuobj, i, 0);
 354                 dev_priv->engine.instmem.flush(dev);
 355         }
 356
 357         spin_lock(&dev_priv->ramin_lock);
 358         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 359         spin_unlock(&dev_priv->ramin_lock);
 360         *pgpuobj = gpuobj;
 361         return 0;
 362 }
 363
 364
 365 static uint32_t
 366 nouveau_gpuobj_class_instmem_size(struct drm_device *dev, int class)
 367 {
 368         struct drm_nouveau_private *dev_priv = dev->dev_private;
 369
 370         /*XXX: dodgy hack for now */
 371         if (dev_priv->card_type >= NV_50)
 372                 return 24;
 373         if (dev_priv->card_type >= NV_40)
 374                 return 32;
 375         return 16;
 376 }
 377
 378 /*
 379    DMA objects are used to reference a piece of memory in the
 380    framebuffer, PCI or AGP address space. Each object is 16 bytes big
 381    and looks as follows:
 382
 383    entry[0]
 384    11:0  class (seems like I can always use 0 here)
 385    12    page table present?
 386    13    page entry linear?
 387    15:14 access: 0 rw, 1 ro, 2 wo
 388    17:16 target: 0 NV memory, 1 NV memory tiled, 2 PCI, 3 AGP
 389    31:20 dma adjust (bits 0-11 of the address)
 390    entry[1]
 391    dma limit (size of transfer)
 392    entry[X]
 393    1     0 readonly, 1 readwrite
 394    31:12 dma frame address of the page (bits 12-31 of the address)
 395    entry[N]
 396    page table terminator, same value as the first pte, as does nvidia
 397    rivatv uses 0xffffffff
 398
 399    Non linear page tables need a list of frame addresses afterwards,
 400    the rivatv project has some info on this.
 401
 402    The method below creates a DMA object in instance RAM and returns a handle
 403    to it that can be used to set up context objects.
 404 */
 405
 406 void
 407 nv50_gpuobj_dma_init(struct nouveau_gpuobj *obj, u32 offset, int class,
 408                      u64 base, u64 size, int target, int access,
 409                      u32 type, u32 comp)
 410 {
 411         struct drm_nouveau_private *dev_priv = obj->dev->dev_private;
 412         struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
 413         u32 flags0;
 414
 415         flags0  = (comp << 29) | (type << 22) | class;
 416         flags0 |= 0x00100000;
 417
 418         switch (access) {
 419         case NV_MEM_ACCESS_RO: flags0 |= 0x00040000; break;
 420         case NV_MEM_ACCESS_RW:
 421         case NV_MEM_ACCESS_WO: flags0 |= 0x00080000; break;
 422         default:
 423                 break;
 424         }
 425
 426         switch (target) {
 427         case NV_MEM_TARGET_VRAM:
 428                 flags0 |= 0x00010000;
 429                 break;
 430         case NV_MEM_TARGET_PCI:
 431                 flags0 |= 0x00020000;
 432                 break;
 433         case NV_MEM_TARGET_PCI_NOSNOOP:
 434                 flags0 |= 0x00030000;
 435                 break;
 436         case NV_MEM_TARGET_GART:
 437                 base += dev_priv->gart_info.aper_base;
 438         default:
 439                 flags0 &= ~0x00100000;
 440                 break;
 441         }
 442
 443         /* convert to base + limit */
 444         size = (base + size) - 1;
 445
 446         nv_wo32(obj, offset + 0x00, flags0);
 447         nv_wo32(obj, offset + 0x04, lower_32_bits(size));
 448         nv_wo32(obj, offset + 0x08, lower_32_bits(base));
 449         nv_wo32(obj, offset + 0x0c, upper_32_bits(size) << 24 |
 450                                     upper_32_bits(base));
 451         nv_wo32(obj, offset + 0x10, 0x00000000);
 452         nv_wo32(obj, offset + 0x14, 0x00000000);
 453
 454         pinstmem->flush(obj->dev);
 455 }
 456
 457 int
 458 nv50_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base, u64 size,
 459                     int target, int access, u32 type, u32 comp,
 460                     struct nouveau_gpuobj **pobj)
 461 {
 462         struct drm_device *dev = chan->dev;
 463         int ret;
 464
 465         ret = nouveau_gpuobj_new(dev, chan, 24, 16, NVOBJ_FLAG_ZERO_FREE, pobj);
 466         if (ret)
 467                 return ret;
 468
 469         nv50_gpuobj_dma_init(*pobj, 0, class, base, size, target,
 470                              access, type, comp);
 471         return 0;
 472 }
 473
 474 int
 475 nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base,
 476                        u64 size, int access, int target,
 477                        struct nouveau_gpuobj **pobj)
 478 {
 479         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 480         struct drm_device *dev = chan->dev;
 481         struct nouveau_gpuobj *obj;
 482         u32 flags0, flags2;
 483         int ret;
 484
 485         if (dev_priv->card_type >= NV_50) {
 486                 u32 comp = (target == NV_MEM_TARGET_VM) ? NV_MEM_COMP_VM : 0;
 487                 u32 type = (target == NV_MEM_TARGET_VM) ? NV_MEM_TYPE_VM : 0;
 488
 489                 return nv50_gpuobj_dma_new(chan, class, base, size,
 490                                            target, access, type, comp, pobj);
 491         }
 492
 493         if (target == NV_MEM_TARGET_GART) {
 494                 struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
 495
 496                 if (dev_priv->gart_info.type == NOUVEAU_GART_PDMA) {
 497                         if (base == 0) {
 498                                 nouveau_gpuobj_ref(gart, pobj);
 499                                 return 0;
 500                         }
 501
 502                         base   = nouveau_sgdma_get_physical(dev, base);
 503                         target = NV_MEM_TARGET_PCI;
 504                 } else {
 505                         base += dev_priv->gart_info.aper_base;
 506                         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP)
 507                                 target = NV_MEM_TARGET_PCI_NOSNOOP;
 508                         else
 509                                 target = NV_MEM_TARGET_PCI;
 510                 }
 511         }
 512
 513         flags0  = class;
 514         flags0 |= 0x00003000; /* PT present, PT linear */
 515         flags2  = 0;
 516
 517         switch (target) {
 518         case NV_MEM_TARGET_PCI:
 519                 flags0 |= 0x00020000;
 520                 break;
 521         case NV_MEM_TARGET_PCI_NOSNOOP:
 522                 flags0 |= 0x00030000;
 523                 break;
 524         default:
 525                 break;
 526         }
 527
 528         switch (access) {
 529         case NV_MEM_ACCESS_RO:
 530                 flags0 |= 0x00004000;
 531                 break;
 532         case NV_MEM_ACCESS_WO:
 533                 flags0 |= 0x00008000;
 534         default:
 535                 flags2 |= 0x00000002;
 536                 break;
 537         }
 538
 539         flags0 |= (base & 0x00000fff) << 20;
 540         flags2 |= (base & 0xfffff000);
 541
 542         ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
 543         if (ret)
 544                 return ret;
 545
 546         nv_wo32(obj, 0x00, flags0);
 547         nv_wo32(obj, 0x04, size - 1);
 548         nv_wo32(obj, 0x08, flags2);
 549         nv_wo32(obj, 0x0c, flags2);
 550
 551         obj->engine = NVOBJ_ENGINE_SW;
 552         obj->class  = class;
 553         *pobj = obj;
 554         return 0;
 555 }
 556
 557 /* Context objects in the instance RAM have the following structure.
 558  * On NV40 they are 32 byte long, on NV30 and smaller 16 bytes.
 559
 560    NV4 - NV30:
 561
 562    entry[0]
 563    11:0 class
 564    12   chroma key enable
 565    13   user clip enable
 566    14   swizzle enable
 567    17:15 patch config:
 568        scrcopy_and, rop_and, blend_and, scrcopy, srccopy_pre, blend_pre
 569    18   synchronize enable
 570    19   endian: 1 big, 0 little
 571    21:20 dither mode
 572    23    single step enable
 573    24    patch status: 0 invalid, 1 valid
 574    25    context_surface 0: 1 valid
 575    26    context surface 1: 1 valid
 576    27    context pattern: 1 valid
 577    28    context rop: 1 valid
 578    29,30 context beta, beta4
 579    entry[1]
 580    7:0   mono format
 581    15:8  color format
 582    31:16 notify instance address
 583    entry[2]
 584    15:0  dma 0 instance address
 585    31:16 dma 1 instance address
 586    entry[3]
 587    dma method traps
 588
 589    NV40:
 590    No idea what the exact format is. Here's what can be deducted:
 591
 592    entry[0]:
 593    11:0  class  (maybe uses more bits here?)
 594    17    user clip enable
 595    21:19 patch config
 596    25    patch status valid ?
 597    entry[1]:
 598    15:0  DMA notifier  (maybe 20:0)
 599    entry[2]:
 600    15:0  DMA 0 instance (maybe 20:0)
 601    24    big endian
 602    entry[3]:
 603    15:0  DMA 1 instance (maybe 20:0)
 604    entry[4]:
 605    entry[5]:
 606    set to 0?
 607 */
 608 static int
 609 nouveau_gpuobj_sw_new(struct nouveau_channel *chan, int class,
 610                       struct nouveau_gpuobj **gpuobj_ret)
 611 {
 612         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 613         struct nouveau_gpuobj *gpuobj;
 614
 615         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 616         if (!gpuobj)
 617                 return -ENOMEM;
 618         gpuobj->dev = chan->dev;
 619         gpuobj->engine = NVOBJ_ENGINE_SW;
 620         gpuobj->class = class;
 621         kref_init(&gpuobj->refcount);
 622         gpuobj->cinst = 0x40;
 623
 624         spin_lock(&dev_priv->ramin_lock);
 625         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 626         spin_unlock(&dev_priv->ramin_lock);
 627         *gpuobj_ret = gpuobj;
 628         return 0;
 629 }
 630
 631 int
 632 nouveau_gpuobj_gr_new(struct nouveau_channel *chan, u32 handle, int class)
 633 {
 634         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 635         struct drm_device *dev = chan->dev;
 636         struct nouveau_gpuobj_class *oc;
 637         struct nouveau_gpuobj *gpuobj;
 638         int ret;
 639
 640         NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
 641
 642         list_for_each_entry(oc, &dev_priv->classes, head) {
 643                 if (oc->id == class)
 644                         goto found;
 645         }
 646
 647         NV_ERROR(dev, "illegal object class: 0x%x\n", class);
 648         return -EINVAL;
 649
 650 found:
 651         switch (oc->engine) {
 652         case NVOBJ_ENGINE_SW:
 653                 if (dev_priv->card_type < NV_C0) {
 654                         ret = nouveau_gpuobj_sw_new(chan, class, &gpuobj);
 655                         if (ret)
 656                                 return ret;
 657                         goto insert;
 658                 }
 659                 break;
 660         case NVOBJ_ENGINE_GR:
 661                 if ((dev_priv->card_type >= NV_20 && !chan->ramin_grctx) ||
 662                     (dev_priv->card_type  < NV_20 && !chan->pgraph_ctx)) {
 663                         struct nouveau_pgraph_engine *pgraph =
 664                                 &dev_priv->engine.graph;
 665
 666                         ret = pgraph->create_context(chan);
 667                         if (ret)
 668                                 return ret;
 669                 }
 670                 break;
 671         case NVOBJ_ENGINE_CRYPT:
 672                 if (!chan->crypt_ctx) {
 673                         struct nouveau_crypt_engine *pcrypt =
 674                                 &dev_priv->engine.crypt;
 675
 676                         ret = pcrypt->create_context(chan);
 677                         if (ret)
 678                                 return ret;
 679                 }
 680                 break;
 681         }
 682
 683         /* we're done if this is fermi */
 684         if (dev_priv->card_type >= NV_C0)
 685                 return 0;
 686
 687         ret = nouveau_gpuobj_new(dev, chan,
 688                                  nouveau_gpuobj_class_instmem_size(dev, class),
 689                                  16,
 690                                  NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
 691                                  &gpuobj);
 692         if (ret) {
 693                 NV_ERROR(dev, "error creating gpuobj: %d\n", ret);
 694                 return ret;
 695         }
 696
 697         if (dev_priv->card_type >= NV_50) {
 698                 nv_wo32(gpuobj,  0, class);
 699                 nv_wo32(gpuobj, 20, 0x00010000);
 700         } else {
 701                 switch (class) {
 702                 case NV_CLASS_NULL:
 703                         nv_wo32(gpuobj, 0, 0x00001030);
 704                         nv_wo32(gpuobj, 4, 0xFFFFFFFF);
 705                         break;
 706                 default:
 707                         if (dev_priv->card_type >= NV_40) {
 708                                 nv_wo32(gpuobj, 0, class);
 709 #ifdef __BIG_ENDIAN
 710                                 nv_wo32(gpuobj, 8, 0x01000000);
 711 #endif
 712                         } else {
 713 #ifdef __BIG_ENDIAN
 714                                 nv_wo32(gpuobj, 0, class | 0x00080000);
 715 #else
 716                                 nv_wo32(gpuobj, 0, class);
 717 #endif
 718                         }
 719                 }
 720         }
 721         dev_priv->engine.instmem.flush(dev);
 722
 723         gpuobj->engine = oc->engine;
 724         gpuobj->class  = oc->id;
 725
 726 insert:
 727         ret = nouveau_ramht_insert(chan, handle, gpuobj);
 728         if (ret)
 729                 NV_ERROR(dev, "error adding gpuobj to RAMHT: %d\n", ret);
 730         nouveau_gpuobj_ref(NULL, &gpuobj);
 731         return ret;
 732 }
 733
 734 static int
 735 nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
 736 {
 737         struct drm_device *dev = chan->dev;
 738         struct drm_nouveau_private *dev_priv = dev->dev_private;
 739         uint32_t size;
 740         uint32_t base;
 741         int ret;
 742
 743         NV_DEBUG(dev, "ch%d\n", chan->id);
 744
 745         /* Base amount for object storage (4KiB enough?) */
 746         size = 0x2000;
 747         base = 0;
 748
 749         /* PGRAPH context */
 750         size += dev_priv->engine.graph.grctx_size;
 751
 752         if (dev_priv->card_type == NV_50) {
 753                 /* Various fixed table thingos */
 754                 size += 0x1400; /* mostly unknown stuff */
 755                 size += 0x4000; /* vm pd */
 756                 base  = 0x6000;
 757                 /* RAMHT, not sure about setting size yet, 32KiB to be safe */
 758                 size += 0x8000;
 759                 /* RAMFC */
 760                 size += 0x1000;
 761         }
 762
 763         ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
 764         if (ret) {
 765                 NV_ERROR(dev, "Error allocating channel PRAMIN: %d\n", ret);
 766                 return ret;
 767         }
 768
 769         ret = drm_mm_init(&chan->ramin_heap, base, size);
 770         if (ret) {
 771                 NV_ERROR(dev, "Error creating PRAMIN heap: %d\n", ret);
 772                 nouveau_gpuobj_ref(NULL, &chan->ramin);
 773                 return ret;
 774         }
 775
 776         return 0;
 777 }
 778
 779 int
 780 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 781                             uint32_t vram_h, uint32_t tt_h)
 782 {
 783         struct drm_device *dev = chan->dev;
 784         struct drm_nouveau_private *dev_priv = dev->dev_private;
 785         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
 786         int ret, i;
 787
 788         NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
 789
 790         if (dev_priv->card_type == NV_C0) {
 791                 struct nouveau_vm *vm = dev_priv->chan_vm;
 792                 struct nouveau_vm_pgd *vpgd;
 793
 794                 ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0,
 795                                          &chan->ramin);
 796                 if (ret)
 797                         return ret;
 798
 799                 nouveau_vm_ref(vm, &chan->vm, NULL);
 800
 801                 vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
 802                 nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
 803                 nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
 804                 nv_wo32(chan->ramin, 0x0208, 0xffffffff);
 805                 nv_wo32(chan->ramin, 0x020c, 0x000000ff);
 806                 return 0;
 807         }
 808
 809         /* Allocate a chunk of memory for per-channel object storage */
 810         ret = nouveau_gpuobj_channel_init_pramin(chan);
 811         if (ret) {
 812                 NV_ERROR(dev, "init pramin\n");
 813                 return ret;
 814         }
 815
 816         /* NV50 VM
 817          *  - Allocate per-channel page-directory
 818          *  - Link with shared channel VM
 819          */
 820         if (dev_priv->chan_vm) {
 821                 u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
 822                 u64 vm_vinst = chan->ramin->vinst + pgd_offs;
 823                 u32 vm_pinst = chan->ramin->pinst;
 824
 825                 if (vm_pinst != ~0)
 826                         vm_pinst += pgd_offs;
 827
 828                 ret = nouveau_gpuobj_new_fake(dev, vm_pinst, vm_vinst, 0x4000,
 829                                               0, &chan->vm_pd);
 830                 if (ret)
 831                         return ret;
 832
 833                 nouveau_vm_ref(dev_priv->chan_vm, &chan->vm, chan->vm_pd);
 834         }
 835
 836         /* RAMHT */
 837         if (dev_priv->card_type < NV_50) {
 838                 nouveau_ramht_ref(dev_priv->ramht, &chan->ramht, NULL);
 839         } else {
 840                 struct nouveau_gpuobj *ramht = NULL;
 841
 842                 ret = nouveau_gpuobj_new(dev, chan, 0x8000, 16,
 843                                          NVOBJ_FLAG_ZERO_ALLOC, &ramht);
 844                 if (ret)
 845                         return ret;
 846
 847                 ret = nouveau_ramht_new(dev, ramht, &chan->ramht);
 848                 nouveau_gpuobj_ref(NULL, &ramht);
 849                 if (ret)
 850                         return ret;
 851
 852                 /* dma objects for display sync channel semaphore blocks */
 853                 for (i = 0; i < 2; i++) {
 854                         struct nouveau_gpuobj *sem = NULL;
 855                         struct nv50_display_crtc *dispc =
 856                                 &nv50_display(dev)->crtc[i];
 857                         u64 offset = dispc->sem.bo->bo.mem.start << PAGE_SHIFT;
 858
 859                         ret = nouveau_gpuobj_dma_new(chan, 0x3d, offset, 0xfff,
 860                                                      NV_MEM_ACCESS_RW,
 861                                                      NV_MEM_TARGET_VRAM, &sem);
 862                         if (ret)
 863                                 return ret;
 864
 865                         ret = nouveau_ramht_insert(chan, NvEvoSema0 + i, sem);
 866                         nouveau_gpuobj_ref(NULL, &sem);
 867                         if (ret)
 868                                 return ret;
 869                 }
 870         }
 871
 872         /* VRAM ctxdma */
 873         if (dev_priv->card_type >= NV_50) {
 874                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 875                                              0, (1ULL << 40), NV_MEM_ACCESS_RW,
 876                                              NV_MEM_TARGET_VM, &vram);
 877                 if (ret) {
 878                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 879                         return ret;
 880                 }
 881         } else {
 882                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 883                                              0, dev_priv->fb_available_size,
 884                                              NV_MEM_ACCESS_RW,
 885                                              NV_MEM_TARGET_VRAM, &vram);
 886                 if (ret) {
 887                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 888                         return ret;
 889                 }
 890         }
 891
 892         ret = nouveau_ramht_insert(chan, vram_h, vram);
 893         nouveau_gpuobj_ref(NULL, &vram);
 894         if (ret) {
 895                 NV_ERROR(dev, "Error adding VRAM ctxdma to RAMHT: %d\n", ret);
 896                 return ret;
 897         }
 898
 899         /* TT memory ctxdma */
 900         if (dev_priv->card_type >= NV_50) {
 901                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 902                                              0, (1ULL << 40), NV_MEM_ACCESS_RW,
 903                                              NV_MEM_TARGET_VM, &tt);
 904         } else {
 905                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 906                                              0, dev_priv->gart_info.aper_size,
 907                                              NV_MEM_ACCESS_RW,
 908                                              NV_MEM_TARGET_GART, &tt);
 909         }
 910
 911         if (ret) {
 912                 NV_ERROR(dev, "Error creating TT ctxdma: %d\n", ret);
 913                 return ret;
 914         }
 915
 916         ret = nouveau_ramht_insert(chan, tt_h, tt);
 917         nouveau_gpuobj_ref(NULL, &tt);
 918         if (ret) {
 919                 NV_ERROR(dev, "Error adding TT ctxdma to RAMHT: %d\n", ret);
 920                 return ret;
 921         }
 922
 923         return 0;
 924 }
 925
 926 void
 927 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
 928 {
 929         struct drm_device *dev = chan->dev;
 930
 931         NV_DEBUG(dev, "ch%d\n", chan->id);
 932
 933         nouveau_ramht_ref(NULL, &chan->ramht, chan);
 934
 935         nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
 936         nouveau_gpuobj_ref(NULL, &chan->vm_pd);
 937
 938         if (drm_mm_initialized(&chan->ramin_heap))
 939                 drm_mm_takedown(&chan->ramin_heap);
 940         nouveau_gpuobj_ref(NULL, &chan->ramin);
 941 }
 942
 943 int
 944 nouveau_gpuobj_suspend(struct drm_device *dev)
 945 {
 946         struct drm_nouveau_private *dev_priv = dev->dev_private;
 947         struct nouveau_gpuobj *gpuobj;
 948         int i;
 949
 950         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 951                 if (gpuobj->cinst != NVOBJ_CINST_GLOBAL)
 952                         continue;
 953
 954                 gpuobj->suspend = vmalloc(gpuobj->size);
 955                 if (!gpuobj->suspend) {
 956                         nouveau_gpuobj_resume(dev);
 957                         return -ENOMEM;
 958                 }
 959
 960                 for (i = 0; i < gpuobj->size; i += 4)
 961                         gpuobj->suspend[i/4] = nv_ro32(gpuobj, i);
 962         }
 963
 964         return 0;
 965 }
 966
 967 void
 968 nouveau_gpuobj_resume(struct drm_device *dev)
 969 {
 970         struct drm_nouveau_private *dev_priv = dev->dev_private;
 971         struct nouveau_gpuobj *gpuobj;
 972         int i;
 973
 974         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 975                 if (!gpuobj->suspend)
 976                         continue;
 977
 978                 for (i = 0; i < gpuobj->size; i += 4)
 979                         nv_wo32(gpuobj, i, gpuobj->suspend[i/4]);
 980
 981                 vfree(gpuobj->suspend);
 982                 gpuobj->suspend = NULL;
 983         }
 984
 985         dev_priv->engine.instmem.flush(dev);
 986 }
 987
 988 int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
 989                               struct drm_file *file_priv)
 990 {
 991         struct drm_nouveau_grobj_alloc *init = data;
 992         struct nouveau_channel *chan;
 993         int ret;
 994
 995         if (init->handle == ~0)
 996                 return -EINVAL;
 997
 998         chan = nouveau_channel_get(dev, file_priv, init->channel);
 999         if (IS_ERR(chan))
1000                 return PTR_ERR(chan);
1001
1002         if (nouveau_ramht_find(chan, init->handle)) {
1003                 ret = -EEXIST;
1004                 goto out;
1005         }
1006
1007         ret = nouveau_gpuobj_gr_new(chan, init->handle, init->class);
1008         if (ret) {
1009                 NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n",
1010                          ret, init->channel, init->handle);
1011         }
1012
1013 out:
1014         nouveau_channel_put(&chan);
1015         return ret;
1016 }
1017
1018 int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
1019                               struct drm_file *file_priv)
1020 {
1021         struct drm_nouveau_gpuobj_free *objfree = data;
1022         struct nouveau_channel *chan;
1023         int ret;
1024
1025         chan = nouveau_channel_get(dev, file_priv, objfree->channel);
1026         if (IS_ERR(chan))
1027                 return PTR_ERR(chan);
1028
1029         /* Synchronize with the user channel */
1030         nouveau_channel_idle(chan);
1031
1032         ret = nouveau_ramht_remove(chan, objfree->handle);
1033         nouveau_channel_put(&chan);
1034         return ret;
1035 }
1036
1037 u32
1038 nv_ro32(struct nouveau_gpuobj *gpuobj, u32 offset)
1039 {
1040         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
1041         struct drm_device *dev = gpuobj->dev;
1042         unsigned long flags;
1043
1044         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
1045                 u64  ptr = gpuobj->vinst + offset;
1046                 u32 base = ptr >> 16;
1047                 u32  val;
1048
1049                 spin_lock_irqsave(&dev_priv->vm_lock, flags);
1050                 if (dev_priv->ramin_base != base) {
1051                         dev_priv->ramin_base = base;
1052                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
1053                 }
1054                 val = nv_rd32(dev, 0x700000 + (ptr & 0xffff));
1055                 spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
1056                 return val;
1057         }
1058
1059         return nv_ri32(dev, gpuobj->pinst + offset);
1060 }
1061
1062 void
1063 nv_wo32(struct nouveau_gpuobj *gpuobj, u32 offset, u32 val)
1064 {
1065         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
1066         struct drm_device *dev = gpuobj->dev;
1067         unsigned long flags;
1068
1069         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
1070                 u64  ptr = gpuobj->vinst + offset;
1071                 u32 base = ptr >> 16;
1072
1073                 spin_lock_irqsave(&dev_priv->vm_lock, flags);
1074                 if (dev_priv->ramin_base != base) {
1075                         dev_priv->ramin_base = base;
1076                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
1077                 }
1078                 nv_wr32(dev, 0x700000 + (ptr & 0xffff), val);
1079                 spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
1080                 return;
1081         }
1082
1083         nv_wi32(dev, gpuobj->pinst + offset, val);
1084 }