drivers/gpu/drm/nouveau/nouveau_object.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drv.h"
  36 #include "nouveau_drm.h"
  37 #include "nouveau_ramht.h"
  38
  39 /* NVidia uses context objects to drive drawing operations.
  40
  41    Context objects can be selected into 8 subchannels in the FIFO,
  42    and then used via DMA command buffers.
  43
  44    A context object is referenced by a user defined handle (CARD32). The HW
  45    looks up graphics objects in a hash table in the instance RAM.
  46
  47    An entry in the hash table consists of 2 CARD32. The first CARD32 contains
  48    the handle, the second one a bitfield, that contains the address of the
  49    object in instance RAM.
  50
  51    The format of the second CARD32 seems to be:
  52
  53    NV4 to NV30:
  54
  55    15: 0  instance_addr >> 4
  56    17:16  engine (here uses 1 = graphics)
  57    28:24  channel id (here uses 0)
  58    31     valid (use 1)
  59
  60    NV40:
  61
  62    15: 0  instance_addr >> 4   (maybe 19-0)
  63    21:20  engine (here uses 1 = graphics)
  64    I'm unsure about the other bits, but using 0 seems to work.
  65
  66    The key into the hash table depends on the object handle and channel id and
  67    is given as:
  68 */
  69
  70 int
  71 nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
  72                    uint32_t size, int align, uint32_t flags,
  73                    struct nouveau_gpuobj **gpuobj_ret)
  74 {
  75         struct drm_nouveau_private *dev_priv = dev->dev_private;
  76         struct nouveau_engine *engine = &dev_priv->engine;
  77         struct nouveau_gpuobj *gpuobj;
  78         struct drm_mm_node *ramin = NULL;
  79         int ret;
  80
  81         NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n",
  82                  chan ? chan->id : -1, size, align, flags);
  83
  84         if (!dev_priv || !gpuobj_ret || *gpuobj_ret != NULL)
  85                 return -EINVAL;
  86
  87         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
  88         if (!gpuobj)
  89                 return -ENOMEM;
  90         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
  91         gpuobj->dev = dev;
  92         gpuobj->flags = flags;
  93         kref_init(&gpuobj->refcount);
  94         gpuobj->size = size;
  95
  96         spin_lock(&dev_priv->ramin_lock);
  97         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
  98         spin_unlock(&dev_priv->ramin_lock);
  99
 100         if (chan) {
 101                 NV_DEBUG(dev, "channel heap\n");
 102
 103                 ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
 104                 if (ramin)
 105                         ramin = drm_mm_get_block(ramin, size, align);
 106
 107                 if (!ramin) {
 108                         nouveau_gpuobj_ref(NULL, &gpuobj);
 109                         return -ENOMEM;
 110                 }
 111         } else {
 112                 NV_DEBUG(dev, "global heap\n");
 113
 114                 /* allocate backing pages, sets vinst */
 115                 ret = engine->instmem.populate(dev, gpuobj, &size);
 116                 if (ret) {
 117                         nouveau_gpuobj_ref(NULL, &gpuobj);
 118                         return ret;
 119                 }
 120
 121                 /* try and get aperture space */
 122                 do {
 123                         if (drm_mm_pre_get(&dev_priv->ramin_heap))
 124                                 return -ENOMEM;
 125
 126                         spin_lock(&dev_priv->ramin_lock);
 127                         ramin = drm_mm_search_free(&dev_priv->ramin_heap, size,
 128                                                    align, 0);
 129                         if (ramin == NULL) {
 130                                 spin_unlock(&dev_priv->ramin_lock);
 131                                 nouveau_gpuobj_ref(NULL, &gpuobj);
 132                                 return -ENOMEM;
 133                         }
 134
 135                         ramin = drm_mm_get_block_atomic(ramin, size, align);
 136                         spin_unlock(&dev_priv->ramin_lock);
 137                 } while (ramin == NULL);
 138
 139                 /* on nv50 it's ok to fail, we have a fallback path */
 140                 if (!ramin && dev_priv->card_type < NV_50) {
 141                         nouveau_gpuobj_ref(NULL, &gpuobj);
 142                         return -ENOMEM;
 143                 }
 144         }
 145
 146         /* if we got a chunk of the aperture, map pages into it */
 147         gpuobj->im_pramin = ramin;
 148         if (!chan && gpuobj->im_pramin && dev_priv->ramin_available) {
 149                 ret = engine->instmem.bind(dev, gpuobj);
 150                 if (ret) {
 151                         nouveau_gpuobj_ref(NULL, &gpuobj);
 152                         return ret;
 153                 }
 154         }
 155
 156         /* calculate the various different addresses for the object */
 157         if (chan) {
 158                 gpuobj->pinst = chan->ramin->pinst;
 159                 if (gpuobj->pinst != ~0)
 160                         gpuobj->pinst += gpuobj->im_pramin->start;
 161
 162                 if (dev_priv->card_type < NV_50) {
 163                         gpuobj->cinst = gpuobj->pinst;
 164                 } else {
 165                         gpuobj->cinst = gpuobj->im_pramin->start;
 166                         gpuobj->vinst = gpuobj->im_pramin->start +
 167                                         chan->ramin->vinst;
 168                 }
 169         } else {
 170                 if (gpuobj->im_pramin)
 171                         gpuobj->pinst = gpuobj->im_pramin->start;
 172                 else
 173                         gpuobj->pinst = ~0;
 174                 gpuobj->cinst = 0xdeadbeef;
 175         }
 176
 177         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 178                 int i;
 179
 180                 for (i = 0; i < gpuobj->size; i += 4)
 181                         nv_wo32(gpuobj, i, 0);
 182                 engine->instmem.flush(dev);
 183         }
 184
 185
 186         *gpuobj_ret = gpuobj;
 187         return 0;
 188 }
 189
 190 int
 191 nouveau_gpuobj_init(struct drm_device *dev)
 192 {
 193         struct drm_nouveau_private *dev_priv = dev->dev_private;
 194
 195         NV_DEBUG(dev, "\n");
 196
 197         INIT_LIST_HEAD(&dev_priv->gpuobj_list);
 198         spin_lock_init(&dev_priv->ramin_lock);
 199         dev_priv->ramin_base = ~0;
 200
 201         return 0;
 202 }
 203
 204 void
 205 nouveau_gpuobj_takedown(struct drm_device *dev)
 206 {
 207         struct drm_nouveau_private *dev_priv = dev->dev_private;
 208
 209         NV_DEBUG(dev, "\n");
 210
 211         BUG_ON(!list_empty(&dev_priv->gpuobj_list));
 212 }
 213
 214
 215 static void
 216 nouveau_gpuobj_del(struct kref *ref)
 217 {
 218         struct nouveau_gpuobj *gpuobj =
 219                 container_of(ref, struct nouveau_gpuobj, refcount);
 220         struct drm_device *dev = gpuobj->dev;
 221         struct drm_nouveau_private *dev_priv = dev->dev_private;
 222         struct nouveau_engine *engine = &dev_priv->engine;
 223         int i;
 224
 225         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 226
 227         if (gpuobj->im_pramin && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
 228                 for (i = 0; i < gpuobj->size; i += 4)
 229                         nv_wo32(gpuobj, i, 0);
 230                 engine->instmem.flush(dev);
 231         }
 232
 233         if (gpuobj->dtor)
 234                 gpuobj->dtor(dev, gpuobj);
 235
 236         if (gpuobj->im_backing)
 237                 engine->instmem.clear(dev, gpuobj);
 238
 239         spin_lock(&dev_priv->ramin_lock);
 240         if (gpuobj->im_pramin)
 241                 drm_mm_put_block(gpuobj->im_pramin);
 242         list_del(&gpuobj->list);
 243         spin_unlock(&dev_priv->ramin_lock);
 244
 245         kfree(gpuobj);
 246 }
 247
 248 void
 249 nouveau_gpuobj_ref(struct nouveau_gpuobj *ref, struct nouveau_gpuobj **ptr)
 250 {
 251         if (ref)
 252                 kref_get(&ref->refcount);
 253
 254         if (*ptr)
 255                 kref_put(&(*ptr)->refcount, nouveau_gpuobj_del);
 256
 257         *ptr = ref;
 258 }
 259
 260 int
 261 nouveau_gpuobj_new_fake(struct drm_device *dev, u32 pinst, u64 vinst,
 262                         u32 size, u32 flags, struct nouveau_gpuobj **pgpuobj)
 263 {
 264         struct drm_nouveau_private *dev_priv = dev->dev_private;
 265         struct nouveau_gpuobj *gpuobj = NULL;
 266         int i;
 267
 268         NV_DEBUG(dev,
 269                  "pinst=0x%08x vinst=0x%010llx size=0x%08x flags=0x%08x\n",
 270                  pinst, vinst, size, flags);
 271
 272         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 273         if (!gpuobj)
 274                 return -ENOMEM;
 275         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 276         gpuobj->dev = dev;
 277         gpuobj->flags = flags;
 278         kref_init(&gpuobj->refcount);
 279         gpuobj->size  = size;
 280         gpuobj->pinst = pinst;
 281         gpuobj->cinst = 0xdeadbeef;
 282         gpuobj->vinst = vinst;
 283
 284         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 285                 for (i = 0; i < gpuobj->size; i += 4)
 286                         nv_wo32(gpuobj, i, 0);
 287                 dev_priv->engine.instmem.flush(dev);
 288         }
 289
 290         spin_lock(&dev_priv->ramin_lock);
 291         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 292         spin_unlock(&dev_priv->ramin_lock);
 293         *pgpuobj = gpuobj;
 294         return 0;
 295 }
 296
 297
 298 static uint32_t
 299 nouveau_gpuobj_class_instmem_size(struct drm_device *dev, int class)
 300 {
 301         struct drm_nouveau_private *dev_priv = dev->dev_private;
 302
 303         /*XXX: dodgy hack for now */
 304         if (dev_priv->card_type >= NV_50)
 305                 return 24;
 306         if (dev_priv->card_type >= NV_40)
 307                 return 32;
 308         return 16;
 309 }
 310
 311 /*
 312    DMA objects are used to reference a piece of memory in the
 313    framebuffer, PCI or AGP address space. Each object is 16 bytes big
 314    and looks as follows:
 315
 316    entry[0]
 317    11:0  class (seems like I can always use 0 here)
 318    12    page table present?
 319    13    page entry linear?
 320    15:14 access: 0 rw, 1 ro, 2 wo
 321    17:16 target: 0 NV memory, 1 NV memory tiled, 2 PCI, 3 AGP
 322    31:20 dma adjust (bits 0-11 of the address)
 323    entry[1]
 324    dma limit (size of transfer)
 325    entry[X]
 326    1     0 readonly, 1 readwrite
 327    31:12 dma frame address of the page (bits 12-31 of the address)
 328    entry[N]
 329    page table terminator, same value as the first pte, as does nvidia
 330    rivatv uses 0xffffffff
 331
 332    Non linear page tables need a list of frame addresses afterwards,
 333    the rivatv project has some info on this.
 334
 335    The method below creates a DMA object in instance RAM and returns a handle
 336    to it that can be used to set up context objects.
 337 */
 338 int
 339 nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class,
 340                        uint64_t offset, uint64_t size, int access,
 341                        int target, struct nouveau_gpuobj **gpuobj)
 342 {
 343         struct drm_device *dev = chan->dev;
 344         struct drm_nouveau_private *dev_priv = dev->dev_private;
 345         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 346         int ret;
 347
 348         NV_DEBUG(dev, "ch%d class=0x%04x offset=0x%llx size=0x%llx\n",
 349                  chan->id, class, offset, size);
 350         NV_DEBUG(dev, "access=%d target=%d\n", access, target);
 351
 352         switch (target) {
 353         case NV_DMA_TARGET_AGP:
 354                 offset += dev_priv->gart_info.aper_base;
 355                 break;
 356         default:
 357                 break;
 358         }
 359
 360         ret = nouveau_gpuobj_new(dev, chan,
 361                                  nouveau_gpuobj_class_instmem_size(dev, class),
 362                                  16, NVOBJ_FLAG_ZERO_ALLOC |
 363                                  NVOBJ_FLAG_ZERO_FREE, gpuobj);
 364         if (ret) {
 365                 NV_ERROR(dev, "Error creating gpuobj: %d\n", ret);
 366                 return ret;
 367         }
 368
 369         if (dev_priv->card_type < NV_50) {
 370                 uint32_t frame, adjust, pte_flags = 0;
 371
 372                 if (access != NV_DMA_ACCESS_RO)
 373                         pte_flags |= (1<<1);
 374                 adjust = offset &  0x00000fff;
 375                 frame  = offset & ~0x00000fff;
 376
 377                 nv_wo32(*gpuobj,  0, ((1<<12) | (1<<13) | (adjust << 20) |
 378                                       (access << 14) | (target << 16) |
 379                                       class));
 380                 nv_wo32(*gpuobj,  4, size - 1);
 381                 nv_wo32(*gpuobj,  8, frame | pte_flags);
 382                 nv_wo32(*gpuobj, 12, frame | pte_flags);
 383         } else {
 384                 uint64_t limit = offset + size - 1;
 385                 uint32_t flags0, flags5;
 386
 387                 if (target == NV_DMA_TARGET_VIDMEM) {
 388                         flags0 = 0x00190000;
 389                         flags5 = 0x00010000;
 390                 } else {
 391                         flags0 = 0x7fc00000;
 392                         flags5 = 0x00080000;
 393                 }
 394
 395                 nv_wo32(*gpuobj,  0, flags0 | class);
 396                 nv_wo32(*gpuobj,  4, lower_32_bits(limit));
 397                 nv_wo32(*gpuobj,  8, lower_32_bits(offset));
 398                 nv_wo32(*gpuobj, 12, ((upper_32_bits(limit) & 0xff) << 24) |
 399                                       (upper_32_bits(offset) & 0xff));
 400                 nv_wo32(*gpuobj, 20, flags5);
 401         }
 402
 403         instmem->flush(dev);
 404
 405         (*gpuobj)->engine = NVOBJ_ENGINE_SW;
 406         (*gpuobj)->class  = class;
 407         return 0;
 408 }
 409
 410 int
 411 nouveau_gpuobj_gart_dma_new(struct nouveau_channel *chan,
 412                             uint64_t offset, uint64_t size, int access,
 413                             struct nouveau_gpuobj **gpuobj,
 414                             uint32_t *o_ret)
 415 {
 416         struct drm_device *dev = chan->dev;
 417         struct drm_nouveau_private *dev_priv = dev->dev_private;
 418         int ret;
 419
 420         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP ||
 421             (dev_priv->card_type >= NV_50 &&
 422              dev_priv->gart_info.type == NOUVEAU_GART_SGDMA)) {
 423                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 424                                              offset + dev_priv->vm_gart_base,
 425                                              size, access, NV_DMA_TARGET_AGP,
 426                                              gpuobj);
 427                 if (o_ret)
 428                         *o_ret = 0;
 429         } else
 430         if (dev_priv->gart_info.type == NOUVEAU_GART_SGDMA) {
 431                 nouveau_gpuobj_ref(dev_priv->gart_info.sg_ctxdma, gpuobj);
 432                 if (offset & ~0xffffffffULL) {
 433                         NV_ERROR(dev, "obj offset exceeds 32-bits\n");
 434                         return -EINVAL;
 435                 }
 436                 if (o_ret)
 437                         *o_ret = (uint32_t)offset;
 438                 ret = (*gpuobj != NULL) ? 0 : -EINVAL;
 439         } else {
 440                 NV_ERROR(dev, "Invalid GART type %d\n", dev_priv->gart_info.type);
 441                 return -EINVAL;
 442         }
 443
 444         return ret;
 445 }
 446
 447 /* Context objects in the instance RAM have the following structure.
 448  * On NV40 they are 32 byte long, on NV30 and smaller 16 bytes.
 449
 450    NV4 - NV30:
 451
 452    entry[0]
 453    11:0 class
 454    12   chroma key enable
 455    13   user clip enable
 456    14   swizzle enable
 457    17:15 patch config:
 458        scrcopy_and, rop_and, blend_and, scrcopy, srccopy_pre, blend_pre
 459    18   synchronize enable
 460    19   endian: 1 big, 0 little
 461    21:20 dither mode
 462    23    single step enable
 463    24    patch status: 0 invalid, 1 valid
 464    25    context_surface 0: 1 valid
 465    26    context surface 1: 1 valid
 466    27    context pattern: 1 valid
 467    28    context rop: 1 valid
 468    29,30 context beta, beta4
 469    entry[1]
 470    7:0   mono format
 471    15:8  color format
 472    31:16 notify instance address
 473    entry[2]
 474    15:0  dma 0 instance address
 475    31:16 dma 1 instance address
 476    entry[3]
 477    dma method traps
 478
 479    NV40:
 480    No idea what the exact format is. Here's what can be deducted:
 481
 482    entry[0]:
 483    11:0  class  (maybe uses more bits here?)
 484    17    user clip enable
 485    21:19 patch config
 486    25    patch status valid ?
 487    entry[1]:
 488    15:0  DMA notifier  (maybe 20:0)
 489    entry[2]:
 490    15:0  DMA 0 instance (maybe 20:0)
 491    24    big endian
 492    entry[3]:
 493    15:0  DMA 1 instance (maybe 20:0)
 494    entry[4]:
 495    entry[5]:
 496    set to 0?
 497 */
 498 int
 499 nouveau_gpuobj_gr_new(struct nouveau_channel *chan, int class,
 500                       struct nouveau_gpuobj **gpuobj)
 501 {
 502         struct drm_device *dev = chan->dev;
 503         struct drm_nouveau_private *dev_priv = dev->dev_private;
 504         int ret;
 505
 506         NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
 507
 508         ret = nouveau_gpuobj_new(dev, chan,
 509                                  nouveau_gpuobj_class_instmem_size(dev, class),
 510                                  16,
 511                                  NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
 512                                  gpuobj);
 513         if (ret) {
 514                 NV_ERROR(dev, "Error creating gpuobj: %d\n", ret);
 515                 return ret;
 516         }
 517
 518         if (dev_priv->card_type >= NV_50) {
 519                 nv_wo32(*gpuobj,  0, class);
 520                 nv_wo32(*gpuobj, 20, 0x00010000);
 521         } else {
 522                 switch (class) {
 523                 case NV_CLASS_NULL:
 524                         nv_wo32(*gpuobj, 0, 0x00001030);
 525                         nv_wo32(*gpuobj, 4, 0xFFFFFFFF);
 526                         break;
 527                 default:
 528                         if (dev_priv->card_type >= NV_40) {
 529                                 nv_wo32(*gpuobj, 0, class);
 530 #ifdef __BIG_ENDIAN
 531                                 nv_wo32(*gpuobj, 8, 0x01000000);
 532 #endif
 533                         } else {
 534 #ifdef __BIG_ENDIAN
 535                                 nv_wo32(*gpuobj, 0, class | 0x00080000);
 536 #else
 537                                 nv_wo32(*gpuobj, 0, class);
 538 #endif
 539                         }
 540                 }
 541         }
 542         dev_priv->engine.instmem.flush(dev);
 543
 544         (*gpuobj)->engine = NVOBJ_ENGINE_GR;
 545         (*gpuobj)->class  = class;
 546         return 0;
 547 }
 548
 549 int
 550 nouveau_gpuobj_sw_new(struct nouveau_channel *chan, int class,
 551                       struct nouveau_gpuobj **gpuobj_ret)
 552 {
 553         struct drm_nouveau_private *dev_priv;
 554         struct nouveau_gpuobj *gpuobj;
 555
 556         if (!chan || !gpuobj_ret || *gpuobj_ret != NULL)
 557                 return -EINVAL;
 558         dev_priv = chan->dev->dev_private;
 559
 560         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 561         if (!gpuobj)
 562                 return -ENOMEM;
 563         gpuobj->dev = chan->dev;
 564         gpuobj->engine = NVOBJ_ENGINE_SW;
 565         gpuobj->class = class;
 566         kref_init(&gpuobj->refcount);
 567         gpuobj->cinst = 0x40;
 568
 569         spin_lock(&dev_priv->ramin_lock);
 570         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 571         spin_unlock(&dev_priv->ramin_lock);
 572         *gpuobj_ret = gpuobj;
 573         return 0;
 574 }
 575
 576 static int
 577 nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
 578 {
 579         struct drm_device *dev = chan->dev;
 580         struct drm_nouveau_private *dev_priv = dev->dev_private;
 581         uint32_t size;
 582         uint32_t base;
 583         int ret;
 584
 585         NV_DEBUG(dev, "ch%d\n", chan->id);
 586
 587         /* Base amount for object storage (4KiB enough?) */
 588         size = 0x1000;
 589         base = 0;
 590
 591         /* PGRAPH context */
 592         size += dev_priv->engine.graph.grctx_size;
 593
 594         if (dev_priv->card_type == NV_50) {
 595                 /* Various fixed table thingos */
 596                 size += 0x1400; /* mostly unknown stuff */
 597                 size += 0x4000; /* vm pd */
 598                 base  = 0x6000;
 599                 /* RAMHT, not sure about setting size yet, 32KiB to be safe */
 600                 size += 0x8000;
 601                 /* RAMFC */
 602                 size += 0x1000;
 603         }
 604
 605         ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
 606         if (ret) {
 607                 NV_ERROR(dev, "Error allocating channel PRAMIN: %d\n", ret);
 608                 return ret;
 609         }
 610
 611         ret = drm_mm_init(&chan->ramin_heap, base, size);
 612         if (ret) {
 613                 NV_ERROR(dev, "Error creating PRAMIN heap: %d\n", ret);
 614                 nouveau_gpuobj_ref(NULL, &chan->ramin);
 615                 return ret;
 616         }
 617
 618         return 0;
 619 }
 620
 621 int
 622 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 623                             uint32_t vram_h, uint32_t tt_h)
 624 {
 625         struct drm_device *dev = chan->dev;
 626         struct drm_nouveau_private *dev_priv = dev->dev_private;
 627         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 628         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
 629         int ret, i;
 630
 631         NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
 632
 633         /* Allocate a chunk of memory for per-channel object storage */
 634         ret = nouveau_gpuobj_channel_init_pramin(chan);
 635         if (ret) {
 636                 NV_ERROR(dev, "init pramin\n");
 637                 return ret;
 638         }
 639
 640         /* NV50 VM
 641          *  - Allocate per-channel page-directory
 642          *  - Map GART and VRAM into the channel's address space at the
 643          *    locations determined during init.
 644          */
 645         if (dev_priv->card_type >= NV_50) {
 646                 u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
 647                 u64 vm_vinst = chan->ramin->vinst + pgd_offs;
 648                 u32 vm_pinst = chan->ramin->pinst;
 649                 u32 pde;
 650
 651                 if (vm_pinst != ~0)
 652                         vm_pinst += pgd_offs;
 653
 654                 ret = nouveau_gpuobj_new_fake(dev, vm_pinst, vm_vinst, 0x4000,
 655                                               0, &chan->vm_pd);
 656                 if (ret)
 657                         return ret;
 658                 for (i = 0; i < 0x4000; i += 8) {
 659                         nv_wo32(chan->vm_pd, i + 0, 0x00000000);
 660                         nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe);
 661                 }
 662
 663                 nouveau_gpuobj_ref(dev_priv->gart_info.sg_ctxdma,
 664                                    &chan->vm_gart_pt);
 665                 pde = (dev_priv->vm_gart_base / (512*1024*1024)) * 8;
 666                 nv_wo32(chan->vm_pd, pde + 0, chan->vm_gart_pt->vinst | 3);
 667                 nv_wo32(chan->vm_pd, pde + 4, 0x00000000);
 668
 669                 pde = (dev_priv->vm_vram_base / (512*1024*1024)) * 8;
 670                 for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) {
 671                         nouveau_gpuobj_ref(dev_priv->vm_vram_pt[i],
 672                                            &chan->vm_vram_pt[i]);
 673
 674                         nv_wo32(chan->vm_pd, pde + 0,
 675                                 chan->vm_vram_pt[i]->vinst | 0x61);
 676                         nv_wo32(chan->vm_pd, pde + 4, 0x00000000);
 677                         pde += 8;
 678                 }
 679
 680                 instmem->flush(dev);
 681         }
 682
 683         /* RAMHT */
 684         if (dev_priv->card_type < NV_50) {
 685                 nouveau_ramht_ref(dev_priv->ramht, &chan->ramht, NULL);
 686         } else {
 687                 struct nouveau_gpuobj *ramht = NULL;
 688
 689                 ret = nouveau_gpuobj_new(dev, chan, 0x8000, 16,
 690                                          NVOBJ_FLAG_ZERO_ALLOC, &ramht);
 691                 if (ret)
 692                         return ret;
 693
 694                 ret = nouveau_ramht_new(dev, ramht, &chan->ramht);
 695                 nouveau_gpuobj_ref(NULL, &ramht);
 696                 if (ret)
 697                         return ret;
 698         }
 699
 700         /* VRAM ctxdma */
 701         if (dev_priv->card_type >= NV_50) {
 702                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 703                                              0, dev_priv->vm_end,
 704                                              NV_DMA_ACCESS_RW,
 705                                              NV_DMA_TARGET_AGP, &vram);
 706                 if (ret) {
 707                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 708                         return ret;
 709                 }
 710         } else {
 711                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 712                                              0, dev_priv->fb_available_size,
 713                                              NV_DMA_ACCESS_RW,
 714                                              NV_DMA_TARGET_VIDMEM, &vram);
 715                 if (ret) {
 716                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 717                         return ret;
 718                 }
 719         }
 720
 721         ret = nouveau_ramht_insert(chan, vram_h, vram);
 722         nouveau_gpuobj_ref(NULL, &vram);
 723         if (ret) {
 724                 NV_ERROR(dev, "Error adding VRAM ctxdma to RAMHT: %d\n", ret);
 725                 return ret;
 726         }
 727
 728         /* TT memory ctxdma */
 729         if (dev_priv->card_type >= NV_50) {
 730                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 731                                              0, dev_priv->vm_end,
 732                                              NV_DMA_ACCESS_RW,
 733                                              NV_DMA_TARGET_AGP, &tt);
 734                 if (ret) {
 735                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 736                         return ret;
 737                 }
 738         } else
 739         if (dev_priv->gart_info.type != NOUVEAU_GART_NONE) {
 740                 ret = nouveau_gpuobj_gart_dma_new(chan, 0,
 741                                                   dev_priv->gart_info.aper_size,
 742                                                   NV_DMA_ACCESS_RW, &tt, NULL);
 743         } else {
 744                 NV_ERROR(dev, "Invalid GART type %d\n", dev_priv->gart_info.type);
 745                 ret = -EINVAL;
 746         }
 747
 748         if (ret) {
 749                 NV_ERROR(dev, "Error creating TT ctxdma: %d\n", ret);
 750                 return ret;
 751         }
 752
 753         ret = nouveau_ramht_insert(chan, tt_h, tt);
 754         nouveau_gpuobj_ref(NULL, &tt);
 755         if (ret) {
 756                 NV_ERROR(dev, "Error adding TT ctxdma to RAMHT: %d\n", ret);
 757                 return ret;
 758         }
 759
 760         return 0;
 761 }
 762
 763 void
 764 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
 765 {
 766         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 767         struct drm_device *dev = chan->dev;
 768         int i;
 769
 770         NV_DEBUG(dev, "ch%d\n", chan->id);
 771
 772         if (!chan->ramht)
 773                 return;
 774
 775         nouveau_ramht_ref(NULL, &chan->ramht, chan);
 776
 777         nouveau_gpuobj_ref(NULL, &chan->vm_pd);
 778         nouveau_gpuobj_ref(NULL, &chan->vm_gart_pt);
 779         for (i = 0; i < dev_priv->vm_vram_pt_nr; i++)
 780                 nouveau_gpuobj_ref(NULL, &chan->vm_vram_pt[i]);
 781
 782         if (chan->ramin_heap.free_stack.next)
 783                 drm_mm_takedown(&chan->ramin_heap);
 784         nouveau_gpuobj_ref(NULL, &chan->ramin);
 785 }
 786
 787 int
 788 nouveau_gpuobj_suspend(struct drm_device *dev)
 789 {
 790         struct drm_nouveau_private *dev_priv = dev->dev_private;
 791         struct nouveau_gpuobj *gpuobj;
 792         int i;
 793
 794         if (dev_priv->card_type < NV_50) {
 795                 dev_priv->susres.ramin_copy = vmalloc(dev_priv->ramin_rsvd_vram);
 796                 if (!dev_priv->susres.ramin_copy)
 797                         return -ENOMEM;
 798
 799                 for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4)
 800                         dev_priv->susres.ramin_copy[i/4] = nv_ri32(dev, i);
 801                 return 0;
 802         }
 803
 804         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 805                 if (!gpuobj->im_backing)
 806                         continue;
 807
 808                 gpuobj->im_backing_suspend = vmalloc(gpuobj->size);
 809                 if (!gpuobj->im_backing_suspend) {
 810                         nouveau_gpuobj_resume(dev);
 811                         return -ENOMEM;
 812                 }
 813
 814                 for (i = 0; i < gpuobj->size; i += 4)
 815                         gpuobj->im_backing_suspend[i/4] = nv_ro32(gpuobj, i);
 816         }
 817
 818         return 0;
 819 }
 820
 821 void
 822 nouveau_gpuobj_suspend_cleanup(struct drm_device *dev)
 823 {
 824         struct drm_nouveau_private *dev_priv = dev->dev_private;
 825         struct nouveau_gpuobj *gpuobj;
 826
 827         if (dev_priv->card_type < NV_50) {
 828                 vfree(dev_priv->susres.ramin_copy);
 829                 dev_priv->susres.ramin_copy = NULL;
 830                 return;
 831         }
 832
 833         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 834                 if (!gpuobj->im_backing_suspend)
 835                         continue;
 836
 837                 vfree(gpuobj->im_backing_suspend);
 838                 gpuobj->im_backing_suspend = NULL;
 839         }
 840 }
 841
 842 void
 843 nouveau_gpuobj_resume(struct drm_device *dev)
 844 {
 845         struct drm_nouveau_private *dev_priv = dev->dev_private;
 846         struct nouveau_gpuobj *gpuobj;
 847         int i;
 848
 849         if (dev_priv->card_type < NV_50) {
 850                 for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4)
 851                         nv_wi32(dev, i, dev_priv->susres.ramin_copy[i/4]);
 852                 nouveau_gpuobj_suspend_cleanup(dev);
 853                 return;
 854         }
 855
 856         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 857                 if (!gpuobj->im_backing_suspend)
 858                         continue;
 859
 860                 for (i = 0; i < gpuobj->size; i += 4)
 861                         nv_wo32(gpuobj, i, gpuobj->im_backing_suspend[i/4]);
 862                 dev_priv->engine.instmem.flush(dev);
 863         }
 864
 865         nouveau_gpuobj_suspend_cleanup(dev);
 866 }
 867
 868 int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
 869                               struct drm_file *file_priv)
 870 {
 871         struct drm_nouveau_private *dev_priv = dev->dev_private;
 872         struct drm_nouveau_grobj_alloc *init = data;
 873         struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
 874         struct nouveau_pgraph_object_class *grc;
 875         struct nouveau_gpuobj *gr = NULL;
 876         struct nouveau_channel *chan;
 877         int ret;
 878
 879         NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(init->channel, file_priv, chan);
 880
 881         if (init->handle == ~0)
 882                 return -EINVAL;
 883
 884         grc = pgraph->grclass;
 885         while (grc->id) {
 886                 if (grc->id == init->class)
 887                         break;
 888                 grc++;
 889         }
 890
 891         if (!grc->id) {
 892                 NV_ERROR(dev, "Illegal object class: 0x%x\n", init->class);
 893                 return -EPERM;
 894         }
 895
 896         if (nouveau_ramht_find(chan, init->handle))
 897                 return -EEXIST;
 898
 899         if (!grc->software)
 900                 ret = nouveau_gpuobj_gr_new(chan, grc->id, &gr);
 901         else
 902                 ret = nouveau_gpuobj_sw_new(chan, grc->id, &gr);
 903         if (ret) {
 904                 NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n",
 905                          ret, init->channel, init->handle);
 906                 return ret;
 907         }
 908
 909         ret = nouveau_ramht_insert(chan, init->handle, gr);
 910         nouveau_gpuobj_ref(NULL, &gr);
 911         if (ret) {
 912                 NV_ERROR(dev, "Error referencing object: %d (%d/0x%08x)\n",
 913                          ret, init->channel, init->handle);
 914                 return ret;
 915         }
 916
 917         return 0;
 918 }
 919
 920 int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
 921                               struct drm_file *file_priv)
 922 {
 923         struct drm_nouveau_gpuobj_free *objfree = data;
 924         struct nouveau_gpuobj *gpuobj;
 925         struct nouveau_channel *chan;
 926
 927         NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(objfree->channel, file_priv, chan);
 928
 929         gpuobj = nouveau_ramht_find(chan, objfree->handle);
 930         if (!gpuobj)
 931                 return -ENOENT;
 932
 933         nouveau_ramht_remove(chan, objfree->handle);
 934         return 0;
 935 }
 936
 937 u32
 938 nv_ro32(struct nouveau_gpuobj *gpuobj, u32 offset)
 939 {
 940         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
 941         struct drm_device *dev = gpuobj->dev;
 942
 943         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
 944                 u64  ptr = gpuobj->vinst + offset;
 945                 u32 base = ptr >> 16;
 946                 u32  val;
 947
 948                 spin_lock(&dev_priv->ramin_lock);
 949                 if (dev_priv->ramin_base != base) {
 950                         dev_priv->ramin_base = base;
 951                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
 952                 }
 953                 val = nv_rd32(dev, 0x700000 + (ptr & 0xffff));
 954                 spin_unlock(&dev_priv->ramin_lock);
 955                 return val;
 956         }
 957
 958         return nv_ri32(dev, gpuobj->pinst + offset);
 959 }
 960
 961 void
 962 nv_wo32(struct nouveau_gpuobj *gpuobj, u32 offset, u32 val)
 963 {
 964         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
 965         struct drm_device *dev = gpuobj->dev;
 966
 967         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
 968                 u64  ptr = gpuobj->vinst + offset;
 969                 u32 base = ptr >> 16;
 970
 971                 spin_lock(&dev_priv->ramin_lock);
 972                 if (dev_priv->ramin_base != base) {
 973                         dev_priv->ramin_base = base;
 974                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
 975                 }
 976                 nv_wr32(dev, 0x700000 + (ptr & 0xffff), val);
 977                 spin_unlock(&dev_priv->ramin_lock);
 978                 return;
 979         }
 980
 981         nv_wi32(dev, gpuobj->pinst + offset, val);
 982 }