drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c

   1 /*
   2  * Copyright 2012 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24 #include "gk104.h"
  25
  26 #include <core/client.h>
  27 #include <core/engctx.h>
  28 #include <core/enum.h>
  29 #include <core/handle.h>
  30 #include <subdev/bar.h>
  31 #include <subdev/fb.h>
  32 #include <subdev/mmu.h>
  33 #include <subdev/timer.h>
  34
  35 #include <nvif/class.h>
  36 #include <nvif/ioctl.h>
  37 #include <nvif/unpack.h>
  38
  39 #define _(a,b) { (a), ((1ULL << (a)) | (b)) }
  40 static const struct {
  41         u64 subdev;
  42         u64 mask;
  43 } fifo_engine[] = {
  44         _(NVDEV_ENGINE_GR      , (1ULL << NVDEV_ENGINE_SW) |
  45                                  (1ULL << NVDEV_ENGINE_CE2)),
  46         _(NVDEV_ENGINE_MSPDEC  , 0),
  47         _(NVDEV_ENGINE_MSPPP   , 0),
  48         _(NVDEV_ENGINE_MSVLD   , 0),
  49         _(NVDEV_ENGINE_CE0     , 0),
  50         _(NVDEV_ENGINE_CE1     , 0),
  51         _(NVDEV_ENGINE_MSENC   , 0),
  52 };
  53 #undef _
  54 #define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine)
  55
  56 struct gk104_fifo_engn {
  57         struct nvkm_gpuobj *runlist[2];
  58         int cur_runlist;
  59         wait_queue_head_t wait;
  60 };
  61
  62 struct gk104_fifo {
  63         struct nvkm_fifo base;
  64
  65         struct work_struct fault;
  66         u64 mask;
  67
  68         struct gk104_fifo_engn engine[FIFO_ENGINE_NR];
  69         struct {
  70                 struct nvkm_gpuobj *mem;
  71                 struct nvkm_vma bar;
  72         } user;
  73         int spoon_nr;
  74 };
  75
  76 struct gk104_fifo_base {
  77         struct nvkm_fifo_base base;
  78         struct nvkm_gpuobj *pgd;
  79         struct nvkm_vm *vm;
  80 };
  81
  82 struct gk104_fifo_chan {
  83         struct nvkm_fifo_chan base;
  84         u32 engine;
  85         enum {
  86                 STOPPED,
  87                 RUNNING,
  88                 KILLED
  89         } state;
  90 };
  91
  92 /*******************************************************************************
  93  * FIFO channel objects
  94  ******************************************************************************/
  95
  96 static void
  97 gk104_fifo_runlist_update(struct gk104_fifo *fifo, u32 engine)
  98 {
  99         struct gk104_fifo_engn *engn = &fifo->engine[engine];
 100         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 101         struct nvkm_device *device = subdev->device;
 102         struct nvkm_bar *bar = device->bar;
 103         struct nvkm_gpuobj *cur;
 104         int i, p;
 105
 106         mutex_lock(&nv_subdev(fifo)->mutex);
 107         cur = engn->runlist[engn->cur_runlist];
 108         engn->cur_runlist = !engn->cur_runlist;
 109
 110         nvkm_kmap(cur);
 111         for (i = 0, p = 0; i < fifo->base.max; i++) {
 112                 struct gk104_fifo_chan *chan = (void *)fifo->base.channel[i];
 113                 if (chan && chan->state == RUNNING && chan->engine == engine) {
 114                         nvkm_wo32(cur, p + 0, i);
 115                         nvkm_wo32(cur, p + 4, 0x00000000);
 116                         p += 8;
 117                 }
 118         }
 119         bar->flush(bar);
 120         nvkm_done(cur);
 121
 122         nvkm_wr32(device, 0x002270, cur->addr >> 12);
 123         nvkm_wr32(device, 0x002274, (engine << 20) | (p >> 3));
 124
 125         if (wait_event_timeout(engn->wait, !(nvkm_rd32(device, 0x002284 +
 126                                (engine * 0x08)) & 0x00100000),
 127                                 msecs_to_jiffies(2000)) == 0)
 128                 nvkm_error(subdev, "runlist %d update timeout\n", engine);
 129         mutex_unlock(&nv_subdev(fifo)->mutex);
 130 }
 131
 132 static int
 133 gk104_fifo_context_attach(struct nvkm_object *parent,
 134                           struct nvkm_object *object)
 135 {
 136         struct nvkm_bar *bar = nvkm_bar(parent);
 137         struct gk104_fifo_base *base = (void *)parent->parent;
 138         struct nvkm_gpuobj *engn = &base->base.gpuobj;
 139         struct nvkm_engctx *ectx = (void *)object;
 140         u32 addr;
 141         int ret;
 142
 143         switch (nv_engidx(object->engine)) {
 144         case NVDEV_ENGINE_SW   :
 145                 return 0;
 146         case NVDEV_ENGINE_CE0:
 147         case NVDEV_ENGINE_CE1:
 148         case NVDEV_ENGINE_CE2:
 149                 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
 150                 return 0;
 151         case NVDEV_ENGINE_GR    : addr = 0x0210; break;
 152         case NVDEV_ENGINE_MSVLD : addr = 0x0270; break;
 153         case NVDEV_ENGINE_MSPDEC: addr = 0x0250; break;
 154         case NVDEV_ENGINE_MSPPP : addr = 0x0260; break;
 155         default:
 156                 return -EINVAL;
 157         }
 158
 159         if (!ectx->vma.node) {
 160                 ret = nvkm_gpuobj_map_vm(nv_gpuobj(ectx), base->vm,
 161                                          NV_MEM_ACCESS_RW, &ectx->vma);
 162                 if (ret)
 163                         return ret;
 164
 165                 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
 166         }
 167
 168         nvkm_kmap(engn);
 169         nvkm_wo32(engn, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4);
 170         nvkm_wo32(engn, addr + 0x04, upper_32_bits(ectx->vma.offset));
 171         bar->flush(bar);
 172         nvkm_done(engn);
 173         return 0;
 174 }
 175
 176 static int
 177 gk104_fifo_chan_kick(struct gk104_fifo_chan *chan)
 178 {
 179         struct nvkm_object *obj = (void *)chan;
 180         struct gk104_fifo *fifo = (void *)obj->engine;
 181         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 182         struct nvkm_device *device = subdev->device;
 183
 184         nvkm_wr32(device, 0x002634, chan->base.chid);
 185         if (nvkm_msec(device, 2000,
 186                 if (!(nvkm_rd32(device, 0x002634) & 0x00100000))
 187                         break;
 188         ) < 0) {
 189                 nvkm_error(subdev, "channel %d [%s] kick timeout\n",
 190                            chan->base.chid, nvkm_client_name(chan));
 191                 return -EBUSY;
 192         }
 193
 194         return 0;
 195 }
 196
 197 static int
 198 gk104_fifo_context_detach(struct nvkm_object *parent, bool suspend,
 199                           struct nvkm_object *object)
 200 {
 201         struct nvkm_bar *bar = nvkm_bar(parent);
 202         struct gk104_fifo_base *base = (void *)parent->parent;
 203         struct gk104_fifo_chan *chan = (void *)parent;
 204         struct nvkm_gpuobj *engn = &base->base.gpuobj;
 205         u32 addr;
 206         int ret;
 207
 208         switch (nv_engidx(object->engine)) {
 209         case NVDEV_ENGINE_SW    : return 0;
 210         case NVDEV_ENGINE_CE0   :
 211         case NVDEV_ENGINE_CE1   :
 212         case NVDEV_ENGINE_CE2   : addr = 0x0000; break;
 213         case NVDEV_ENGINE_GR    : addr = 0x0210; break;
 214         case NVDEV_ENGINE_MSVLD : addr = 0x0270; break;
 215         case NVDEV_ENGINE_MSPDEC: addr = 0x0250; break;
 216         case NVDEV_ENGINE_MSPPP : addr = 0x0260; break;
 217         default:
 218                 return -EINVAL;
 219         }
 220
 221         ret = gk104_fifo_chan_kick(chan);
 222         if (ret && suspend)
 223                 return ret;
 224
 225         if (addr) {
 226                 nvkm_kmap(engn);
 227                 nvkm_wo32(engn, addr + 0x00, 0x00000000);
 228                 nvkm_wo32(engn, addr + 0x04, 0x00000000);
 229                 bar->flush(bar);
 230                 nvkm_done(engn);
 231         }
 232
 233         return 0;
 234 }
 235
 236 static int
 237 gk104_fifo_chan_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
 238                      struct nvkm_oclass *oclass, void *data, u32 size,
 239                      struct nvkm_object **pobject)
 240 {
 241         union {
 242                 struct kepler_channel_gpfifo_a_v0 v0;
 243         } *args = data;
 244         struct nvkm_bar *bar = nvkm_bar(parent);
 245         struct gk104_fifo *fifo = (void *)engine;
 246         struct gk104_fifo_base *base = (void *)parent;
 247         struct gk104_fifo_chan *chan;
 248         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 249         struct nvkm_gpuobj *ramfc = &base->base.gpuobj;
 250         u64 usermem, ioffset, ilength;
 251         int ret, i;
 252
 253         nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
 254         if (nvif_unpack(args->v0, 0, 0, false)) {
 255                 nvif_ioctl(parent, "create channel gpfifo vers %d pushbuf %llx "
 256                                    "ioffset %016llx ilength %08x engine %08x\n",
 257                            args->v0.version, args->v0.pushbuf, args->v0.ioffset,
 258                            args->v0.ilength, args->v0.engine);
 259         } else
 260                 return ret;
 261
 262         for (i = 0; i < FIFO_ENGINE_NR; i++) {
 263                 if (args->v0.engine & (1 << i)) {
 264                         if (nvkm_engine(parent, fifo_engine[i].subdev)) {
 265                                 args->v0.engine = (1 << i);
 266                                 break;
 267                         }
 268                 }
 269         }
 270
 271         if (i == FIFO_ENGINE_NR) {
 272                 nvkm_error(subdev, "unsupported engines %08x\n",
 273                            args->v0.engine);
 274                 return -ENODEV;
 275         }
 276
 277         ret = nvkm_fifo_channel_create(parent, engine, oclass, 1,
 278                                        fifo->user.bar.offset, 0x200,
 279                                        args->v0.pushbuf,
 280                                        fifo_engine[i].mask, &chan);
 281         *pobject = nv_object(chan);
 282         if (ret)
 283                 return ret;
 284
 285         args->v0.chid = chan->base.chid;
 286
 287         nv_parent(chan)->context_attach = gk104_fifo_context_attach;
 288         nv_parent(chan)->context_detach = gk104_fifo_context_detach;
 289         chan->engine = i;
 290
 291         usermem = chan->base.chid * 0x200;
 292         ioffset = args->v0.ioffset;
 293         ilength = order_base_2(args->v0.ilength / 8);
 294
 295         nvkm_kmap(fifo->user.mem);
 296         for (i = 0; i < 0x200; i += 4)
 297                 nvkm_wo32(fifo->user.mem, usermem + i, 0x00000000);
 298         nvkm_done(fifo->user.mem);
 299
 300         nvkm_kmap(ramfc);
 301         nvkm_wo32(ramfc, 0x08, lower_32_bits(fifo->user.mem->addr + usermem));
 302         nvkm_wo32(ramfc, 0x0c, upper_32_bits(fifo->user.mem->addr + usermem));
 303         nvkm_wo32(ramfc, 0x10, 0x0000face);
 304         nvkm_wo32(ramfc, 0x30, 0xfffff902);
 305         nvkm_wo32(ramfc, 0x48, lower_32_bits(ioffset));
 306         nvkm_wo32(ramfc, 0x4c, upper_32_bits(ioffset) | (ilength << 16));
 307         nvkm_wo32(ramfc, 0x84, 0x20400000);
 308         nvkm_wo32(ramfc, 0x94, 0x30000001);
 309         nvkm_wo32(ramfc, 0x9c, 0x00000100);
 310         nvkm_wo32(ramfc, 0xac, 0x0000001f);
 311         nvkm_wo32(ramfc, 0xe8, chan->base.chid);
 312         nvkm_wo32(ramfc, 0xb8, 0xf8000000);
 313         nvkm_wo32(ramfc, 0xf8, 0x10003080); /* 0x002310 */
 314         nvkm_wo32(ramfc, 0xfc, 0x10000010); /* 0x002350 */
 315         bar->flush(bar);
 316         nvkm_done(ramfc);
 317         return 0;
 318 }
 319
 320 static int
 321 gk104_fifo_chan_init(struct nvkm_object *object)
 322 {
 323         struct nvkm_gpuobj *base = nv_gpuobj(object->parent);
 324         struct gk104_fifo *fifo = (void *)object->engine;
 325         struct gk104_fifo_chan *chan = (void *)object;
 326         struct nvkm_device *device = fifo->base.engine.subdev.device;
 327         u32 chid = chan->base.chid;
 328         int ret;
 329
 330         ret = nvkm_fifo_channel_init(&chan->base);
 331         if (ret)
 332                 return ret;
 333
 334         nvkm_mask(device, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16);
 335         nvkm_wr32(device, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12);
 336
 337         if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) {
 338                 nvkm_mask(device, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
 339                 gk104_fifo_runlist_update(fifo, chan->engine);
 340                 nvkm_mask(device, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
 341         }
 342
 343         return 0;
 344 }
 345
 346 static int
 347 gk104_fifo_chan_fini(struct nvkm_object *object, bool suspend)
 348 {
 349         struct gk104_fifo *fifo = (void *)object->engine;
 350         struct gk104_fifo_chan *chan = (void *)object;
 351         struct nvkm_device *device = fifo->base.engine.subdev.device;
 352         u32 chid = chan->base.chid;
 353
 354         if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) {
 355                 nvkm_mask(device, 0x800004 + (chid * 8), 0x00000800, 0x00000800);
 356                 gk104_fifo_runlist_update(fifo, chan->engine);
 357         }
 358
 359         nvkm_wr32(device, 0x800000 + (chid * 8), 0x00000000);
 360         return nvkm_fifo_channel_fini(&chan->base, suspend);
 361 }
 362
 363 struct nvkm_ofuncs
 364 gk104_fifo_chan_ofuncs = {
 365         .ctor = gk104_fifo_chan_ctor,
 366         .dtor = _nvkm_fifo_channel_dtor,
 367         .init = gk104_fifo_chan_init,
 368         .fini = gk104_fifo_chan_fini,
 369         .map  = _nvkm_fifo_channel_map,
 370         .rd32 = _nvkm_fifo_channel_rd32,
 371         .wr32 = _nvkm_fifo_channel_wr32,
 372         .ntfy = _nvkm_fifo_channel_ntfy
 373 };
 374
 375 static struct nvkm_oclass
 376 gk104_fifo_sclass[] = {
 377         { KEPLER_CHANNEL_GPFIFO_A, &gk104_fifo_chan_ofuncs },
 378         {}
 379 };
 380
 381 /*******************************************************************************
 382  * FIFO context - instmem heap and vm setup
 383  ******************************************************************************/
 384
 385 static int
 386 gk104_fifo_context_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
 387                         struct nvkm_oclass *oclass, void *data, u32 size,
 388                         struct nvkm_object **pobject)
 389 {
 390         struct gk104_fifo_base *base;
 391         int ret;
 392
 393         ret = nvkm_fifo_context_create(parent, engine, oclass, NULL, 0x1000,
 394                                        0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base);
 395         *pobject = nv_object(base);
 396         if (ret)
 397                 return ret;
 398
 399         ret = nvkm_gpuobj_new(nv_object(base), NULL, 0x10000, 0x1000, 0,
 400                               &base->pgd);
 401         if (ret)
 402                 return ret;
 403
 404         nvkm_kmap(&base->base.gpuobj);
 405         nvkm_wo32(&base->base.gpuobj, 0x0200, lower_32_bits(base->pgd->addr));
 406         nvkm_wo32(&base->base.gpuobj, 0x0204, upper_32_bits(base->pgd->addr));
 407         nvkm_wo32(&base->base.gpuobj, 0x0208, 0xffffffff);
 408         nvkm_wo32(&base->base.gpuobj, 0x020c, 0x000000ff);
 409         nvkm_done(&base->base.gpuobj);
 410
 411         ret = nvkm_vm_ref(nvkm_client(parent)->vm, &base->vm, base->pgd);
 412         if (ret)
 413                 return ret;
 414
 415         return 0;
 416 }
 417
 418 static void
 419 gk104_fifo_context_dtor(struct nvkm_object *object)
 420 {
 421         struct gk104_fifo_base *base = (void *)object;
 422         nvkm_vm_ref(NULL, &base->vm, base->pgd);
 423         nvkm_gpuobj_ref(NULL, &base->pgd);
 424         nvkm_fifo_context_destroy(&base->base);
 425 }
 426
 427 static struct nvkm_oclass
 428 gk104_fifo_cclass = {
 429         .handle = NV_ENGCTX(FIFO, 0xe0),
 430         .ofuncs = &(struct nvkm_ofuncs) {
 431                 .ctor = gk104_fifo_context_ctor,
 432                 .dtor = gk104_fifo_context_dtor,
 433                 .init = _nvkm_fifo_context_init,
 434                 .fini = _nvkm_fifo_context_fini,
 435                 .rd32 = _nvkm_fifo_context_rd32,
 436                 .wr32 = _nvkm_fifo_context_wr32,
 437         },
 438 };
 439
 440 /*******************************************************************************
 441  * PFIFO engine
 442  ******************************************************************************/
 443
 444 static inline int
 445 gk104_fifo_engidx(struct gk104_fifo *fifo, u32 engn)
 446 {
 447         switch (engn) {
 448         case NVDEV_ENGINE_GR    :
 449         case NVDEV_ENGINE_CE2   : engn = 0; break;
 450         case NVDEV_ENGINE_MSVLD : engn = 1; break;
 451         case NVDEV_ENGINE_MSPPP : engn = 2; break;
 452         case NVDEV_ENGINE_MSPDEC: engn = 3; break;
 453         case NVDEV_ENGINE_CE0   : engn = 4; break;
 454         case NVDEV_ENGINE_CE1   : engn = 5; break;
 455         case NVDEV_ENGINE_MSENC : engn = 6; break;
 456         default:
 457                 return -1;
 458         }
 459
 460         return engn;
 461 }
 462
 463 static inline struct nvkm_engine *
 464 gk104_fifo_engine(struct gk104_fifo *fifo, u32 engn)
 465 {
 466         if (engn >= ARRAY_SIZE(fifo_engine))
 467                 return NULL;
 468         return nvkm_engine(fifo, fifo_engine[engn].subdev);
 469 }
 470
 471 static void
 472 gk104_fifo_recover_work(struct work_struct *work)
 473 {
 474         struct gk104_fifo *fifo = container_of(work, typeof(*fifo), fault);
 475         struct nvkm_device *device = fifo->base.engine.subdev.device;
 476         struct nvkm_object *engine;
 477         unsigned long flags;
 478         u32 engn, engm = 0;
 479         u64 mask, todo;
 480
 481         spin_lock_irqsave(&fifo->base.lock, flags);
 482         mask = fifo->mask;
 483         fifo->mask = 0ULL;
 484         spin_unlock_irqrestore(&fifo->base.lock, flags);
 485
 486         for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn))
 487                 engm |= 1 << gk104_fifo_engidx(fifo, engn);
 488         nvkm_mask(device, 0x002630, engm, engm);
 489
 490         for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) {
 491                 if ((engine = (void *)nvkm_engine(fifo, engn))) {
 492                         nv_ofuncs(engine)->fini(engine, false);
 493                         WARN_ON(nv_ofuncs(engine)->init(engine));
 494                 }
 495                 gk104_fifo_runlist_update(fifo, gk104_fifo_engidx(fifo, engn));
 496         }
 497
 498         nvkm_wr32(device, 0x00262c, engm);
 499         nvkm_mask(device, 0x002630, engm, 0x00000000);
 500 }
 501
 502 static void
 503 gk104_fifo_recover(struct gk104_fifo *fifo, struct nvkm_engine *engine,
 504                   struct gk104_fifo_chan *chan)
 505 {
 506         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 507         struct nvkm_device *device = subdev->device;
 508         u32 chid = chan->base.chid;
 509         unsigned long flags;
 510
 511         nvkm_error(subdev, "%s engine fault on channel %d, recovering...\n",
 512                    nv_subdev(engine)->name, chid);
 513
 514         nvkm_mask(device, 0x800004 + (chid * 0x08), 0x00000800, 0x00000800);
 515         chan->state = KILLED;
 516
 517         spin_lock_irqsave(&fifo->base.lock, flags);
 518         fifo->mask |= 1ULL << nv_engidx(engine);
 519         spin_unlock_irqrestore(&fifo->base.lock, flags);
 520         schedule_work(&fifo->fault);
 521 }
 522
 523 static int
 524 gk104_fifo_swmthd(struct gk104_fifo *fifo, u32 chid, u32 mthd, u32 data)
 525 {
 526         struct gk104_fifo_chan *chan = NULL;
 527         struct nvkm_handle *bind;
 528         unsigned long flags;
 529         int ret = -EINVAL;
 530
 531         spin_lock_irqsave(&fifo->base.lock, flags);
 532         if (likely(chid >= fifo->base.min && chid <= fifo->base.max))
 533                 chan = (void *)fifo->base.channel[chid];
 534         if (unlikely(!chan))
 535                 goto out;
 536
 537         bind = nvkm_namedb_get_class(nv_namedb(chan), NVIF_IOCTL_NEW_V0_SW_GF100);
 538         if (likely(bind)) {
 539                 if (!mthd || !nv_call(bind->object, mthd, data))
 540                         ret = 0;
 541                 nvkm_namedb_put(bind);
 542         }
 543
 544 out:
 545         spin_unlock_irqrestore(&fifo->base.lock, flags);
 546         return ret;
 547 }
 548
 549 static const struct nvkm_enum
 550 gk104_fifo_bind_reason[] = {
 551         { 0x01, "BIND_NOT_UNBOUND" },
 552         { 0x02, "SNOOP_WITHOUT_BAR1" },
 553         { 0x03, "UNBIND_WHILE_RUNNING" },
 554         { 0x05, "INVALID_RUNLIST" },
 555         { 0x06, "INVALID_CTX_TGT" },
 556         { 0x0b, "UNBIND_WHILE_PARKED" },
 557         {}
 558 };
 559
 560 static void
 561 gk104_fifo_intr_bind(struct gk104_fifo *fifo)
 562 {
 563         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 564         struct nvkm_device *device = subdev->device;
 565         u32 intr = nvkm_rd32(device, 0x00252c);
 566         u32 code = intr & 0x000000ff;
 567         const struct nvkm_enum *en =
 568                 nvkm_enum_find(gk104_fifo_bind_reason, code);
 569
 570         nvkm_error(subdev, "BIND_ERROR %02x [%s]\n", code, en ? en->name : "");
 571 }
 572
 573 static const struct nvkm_enum
 574 gk104_fifo_sched_reason[] = {
 575         { 0x0a, "CTXSW_TIMEOUT" },
 576         {}
 577 };
 578
 579 static void
 580 gk104_fifo_intr_sched_ctxsw(struct gk104_fifo *fifo)
 581 {
 582         struct nvkm_device *device = fifo->base.engine.subdev.device;
 583         struct nvkm_engine *engine;
 584         struct gk104_fifo_chan *chan;
 585         u32 engn;
 586
 587         for (engn = 0; engn < ARRAY_SIZE(fifo_engine); engn++) {
 588                 u32 stat = nvkm_rd32(device, 0x002640 + (engn * 0x04));
 589                 u32 busy = (stat & 0x80000000);
 590                 u32 next = (stat & 0x07ff0000) >> 16;
 591                 u32 chsw = (stat & 0x00008000);
 592                 u32 save = (stat & 0x00004000);
 593                 u32 load = (stat & 0x00002000);
 594                 u32 prev = (stat & 0x000007ff);
 595                 u32 chid = load ? next : prev;
 596                 (void)save;
 597
 598                 if (busy && chsw) {
 599                         if (!(chan = (void *)fifo->base.channel[chid]))
 600                                 continue;
 601                         if (!(engine = gk104_fifo_engine(fifo, engn)))
 602                                 continue;
 603                         gk104_fifo_recover(fifo, engine, chan);
 604                 }
 605         }
 606 }
 607
 608 static void
 609 gk104_fifo_intr_sched(struct gk104_fifo *fifo)
 610 {
 611         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 612         struct nvkm_device *device = subdev->device;
 613         u32 intr = nvkm_rd32(device, 0x00254c);
 614         u32 code = intr & 0x000000ff;
 615         const struct nvkm_enum *en =
 616                 nvkm_enum_find(gk104_fifo_sched_reason, code);
 617
 618         nvkm_error(subdev, "SCHED_ERROR %02x [%s]\n", code, en ? en->name : "");
 619
 620         switch (code) {
 621         case 0x0a:
 622                 gk104_fifo_intr_sched_ctxsw(fifo);
 623                 break;
 624         default:
 625                 break;
 626         }
 627 }
 628
 629 static void
 630 gk104_fifo_intr_chsw(struct gk104_fifo *fifo)
 631 {
 632         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 633         struct nvkm_device *device = subdev->device;
 634         u32 stat = nvkm_rd32(device, 0x00256c);
 635         nvkm_error(subdev, "CHSW_ERROR %08x\n", stat);
 636         nvkm_wr32(device, 0x00256c, stat);
 637 }
 638
 639 static void
 640 gk104_fifo_intr_dropped_fault(struct gk104_fifo *fifo)
 641 {
 642         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 643         struct nvkm_device *device = subdev->device;
 644         u32 stat = nvkm_rd32(device, 0x00259c);
 645         nvkm_error(subdev, "DROPPED_MMU_FAULT %08x\n", stat);
 646 }
 647
 648 static const struct nvkm_enum
 649 gk104_fifo_fault_engine[] = {
 650         { 0x00, "GR", NULL, NVDEV_ENGINE_GR },
 651         { 0x03, "IFB", NULL, NVDEV_ENGINE_IFB },
 652         { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR },
 653         { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM },
 654         { 0x07, "PBDMA0", NULL, NVDEV_ENGINE_FIFO },
 655         { 0x08, "PBDMA1", NULL, NVDEV_ENGINE_FIFO },
 656         { 0x09, "PBDMA2", NULL, NVDEV_ENGINE_FIFO },
 657         { 0x10, "MSVLD", NULL, NVDEV_ENGINE_MSVLD },
 658         { 0x11, "MSPPP", NULL, NVDEV_ENGINE_MSPPP },
 659         { 0x13, "PERF" },
 660         { 0x14, "MSPDEC", NULL, NVDEV_ENGINE_MSPDEC },
 661         { 0x15, "CE0", NULL, NVDEV_ENGINE_CE0 },
 662         { 0x16, "CE1", NULL, NVDEV_ENGINE_CE1 },
 663         { 0x17, "PMU" },
 664         { 0x19, "MSENC", NULL, NVDEV_ENGINE_MSENC },
 665         { 0x1b, "CE2", NULL, NVDEV_ENGINE_CE2 },
 666         {}
 667 };
 668
 669 static const struct nvkm_enum
 670 gk104_fifo_fault_reason[] = {
 671         { 0x00, "PDE" },
 672         { 0x01, "PDE_SIZE" },
 673         { 0x02, "PTE" },
 674         { 0x03, "VA_LIMIT_VIOLATION" },
 675         { 0x04, "UNBOUND_INST_BLOCK" },
 676         { 0x05, "PRIV_VIOLATION" },
 677         { 0x06, "RO_VIOLATION" },
 678         { 0x07, "WO_VIOLATION" },
 679         { 0x08, "PITCH_MASK_VIOLATION" },
 680         { 0x09, "WORK_CREATION" },
 681         { 0x0a, "UNSUPPORTED_APERTURE" },
 682         { 0x0b, "COMPRESSION_FAILURE" },
 683         { 0x0c, "UNSUPPORTED_KIND" },
 684         { 0x0d, "REGION_VIOLATION" },
 685         { 0x0e, "BOTH_PTES_VALID" },
 686         { 0x0f, "INFO_TYPE_POISONED" },
 687         {}
 688 };
 689
 690 static const struct nvkm_enum
 691 gk104_fifo_fault_hubclient[] = {
 692         { 0x00, "VIP" },
 693         { 0x01, "CE0" },
 694         { 0x02, "CE1" },
 695         { 0x03, "DNISO" },
 696         { 0x04, "FE" },
 697         { 0x05, "FECS" },
 698         { 0x06, "HOST" },
 699         { 0x07, "HOST_CPU" },
 700         { 0x08, "HOST_CPU_NB" },
 701         { 0x09, "ISO" },
 702         { 0x0a, "MMU" },
 703         { 0x0b, "MSPDEC" },
 704         { 0x0c, "MSPPP" },
 705         { 0x0d, "MSVLD" },
 706         { 0x0e, "NISO" },
 707         { 0x0f, "P2P" },
 708         { 0x10, "PD" },
 709         { 0x11, "PERF" },
 710         { 0x12, "PMU" },
 711         { 0x13, "RASTERTWOD" },
 712         { 0x14, "SCC" },
 713         { 0x15, "SCC_NB" },
 714         { 0x16, "SEC" },
 715         { 0x17, "SSYNC" },
 716         { 0x18, "GR_CE" },
 717         { 0x19, "CE2" },
 718         { 0x1a, "XV" },
 719         { 0x1b, "MMU_NB" },
 720         { 0x1c, "MSENC" },
 721         { 0x1d, "DFALCON" },
 722         { 0x1e, "SKED" },
 723         { 0x1f, "AFALCON" },
 724         {}
 725 };
 726
 727 static const struct nvkm_enum
 728 gk104_fifo_fault_gpcclient[] = {
 729         { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" },
 730         { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" },
 731         { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" },
 732         { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" },
 733         { 0x0c, "RAST" },
 734         { 0x0d, "GCC" },
 735         { 0x0e, "GPCCS" },
 736         { 0x0f, "PROP_0" },
 737         { 0x10, "PROP_1" },
 738         { 0x11, "PROP_2" },
 739         { 0x12, "PROP_3" },
 740         { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" },
 741         { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" },
 742         { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" },
 743         { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" },
 744         { 0x1f, "GPM" },
 745         { 0x20, "LTP_UTLB_0" },
 746         { 0x21, "LTP_UTLB_1" },
 747         { 0x22, "LTP_UTLB_2" },
 748         { 0x23, "LTP_UTLB_3" },
 749         { 0x24, "GPC_RGG_UTLB" },
 750         {}
 751 };
 752
 753 static void
 754 gk104_fifo_intr_fault(struct gk104_fifo *fifo, int unit)
 755 {
 756         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 757         struct nvkm_device *device = subdev->device;
 758         u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10));
 759         u32 valo = nvkm_rd32(device, 0x002804 + (unit * 0x10));
 760         u32 vahi = nvkm_rd32(device, 0x002808 + (unit * 0x10));
 761         u32 stat = nvkm_rd32(device, 0x00280c + (unit * 0x10));
 762         u32 gpc    = (stat & 0x1f000000) >> 24;
 763         u32 client = (stat & 0x00001f00) >> 8;
 764         u32 write  = (stat & 0x00000080);
 765         u32 hub    = (stat & 0x00000040);
 766         u32 reason = (stat & 0x0000000f);
 767         struct nvkm_object *engctx = NULL, *object;
 768         struct nvkm_engine *engine = NULL;
 769         const struct nvkm_enum *er, *eu, *ec;
 770         char gpcid[8] = "";
 771
 772         er = nvkm_enum_find(gk104_fifo_fault_reason, reason);
 773         eu = nvkm_enum_find(gk104_fifo_fault_engine, unit);
 774         if (hub) {
 775                 ec = nvkm_enum_find(gk104_fifo_fault_hubclient, client);
 776         } else {
 777                 ec = nvkm_enum_find(gk104_fifo_fault_gpcclient, client);
 778                 snprintf(gpcid, sizeof(gpcid), "GPC%d/", gpc);
 779         }
 780
 781         if (eu) {
 782                 switch (eu->data2) {
 783                 case NVDEV_SUBDEV_BAR:
 784                         nvkm_mask(device, 0x001704, 0x00000000, 0x00000000);
 785                         break;
 786                 case NVDEV_SUBDEV_INSTMEM:
 787                         nvkm_mask(device, 0x001714, 0x00000000, 0x00000000);
 788                         break;
 789                 case NVDEV_ENGINE_IFB:
 790                         nvkm_mask(device, 0x001718, 0x00000000, 0x00000000);
 791                         break;
 792                 default:
 793                         engine = nvkm_engine(fifo, eu->data2);
 794                         if (engine)
 795                                 engctx = nvkm_engctx_get(engine, inst);
 796                         break;
 797                 }
 798         }
 799
 800         nvkm_error(subdev,
 801                    "%s fault at %010llx engine %02x [%s] client %02x [%s%s] "
 802                    "reason %02x [%s] on channel %d [%010llx %s]\n",
 803                    write ? "write" : "read", (u64)vahi << 32 | valo,
 804                    unit, eu ? eu->name : "", client, gpcid, ec ? ec->name : "",
 805                    reason, er ? er->name : "", -1, (u64)inst << 12,
 806                    nvkm_client_name(engctx));
 807
 808         object = engctx;
 809         while (object) {
 810                 switch (nv_mclass(object)) {
 811                 case KEPLER_CHANNEL_GPFIFO_A:
 812                 case MAXWELL_CHANNEL_GPFIFO_A:
 813                         gk104_fifo_recover(fifo, engine, (void *)object);
 814                         break;
 815                 }
 816                 object = object->parent;
 817         }
 818
 819         nvkm_engctx_put(engctx);
 820 }
 821
 822 static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = {
 823         { 0x00000001, "MEMREQ" },
 824         { 0x00000002, "MEMACK_TIMEOUT" },
 825         { 0x00000004, "MEMACK_EXTRA" },
 826         { 0x00000008, "MEMDAT_TIMEOUT" },
 827         { 0x00000010, "MEMDAT_EXTRA" },
 828         { 0x00000020, "MEMFLUSH" },
 829         { 0x00000040, "MEMOP" },
 830         { 0x00000080, "LBCONNECT" },
 831         { 0x00000100, "LBREQ" },
 832         { 0x00000200, "LBACK_TIMEOUT" },
 833         { 0x00000400, "LBACK_EXTRA" },
 834         { 0x00000800, "LBDAT_TIMEOUT" },
 835         { 0x00001000, "LBDAT_EXTRA" },
 836         { 0x00002000, "GPFIFO" },
 837         { 0x00004000, "GPPTR" },
 838         { 0x00008000, "GPENTRY" },
 839         { 0x00010000, "GPCRC" },
 840         { 0x00020000, "PBPTR" },
 841         { 0x00040000, "PBENTRY" },
 842         { 0x00080000, "PBCRC" },
 843         { 0x00100000, "XBARCONNECT" },
 844         { 0x00200000, "METHOD" },
 845         { 0x00400000, "METHODCRC" },
 846         { 0x00800000, "DEVICE" },
 847         { 0x02000000, "SEMAPHORE" },
 848         { 0x04000000, "ACQUIRE" },
 849         { 0x08000000, "PRI" },
 850         { 0x20000000, "NO_CTXSW_SEG" },
 851         { 0x40000000, "PBSEG" },
 852         { 0x80000000, "SIGNATURE" },
 853         {}
 854 };
 855
 856 static void
 857 gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit)
 858 {
 859         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 860         struct nvkm_device *device = subdev->device;
 861         u32 mask = nvkm_rd32(device, 0x04010c + (unit * 0x2000));
 862         u32 stat = nvkm_rd32(device, 0x040108 + (unit * 0x2000)) & mask;
 863         u32 addr = nvkm_rd32(device, 0x0400c0 + (unit * 0x2000));
 864         u32 data = nvkm_rd32(device, 0x0400c4 + (unit * 0x2000));
 865         u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff;
 866         u32 subc = (addr & 0x00070000) >> 16;
 867         u32 mthd = (addr & 0x00003ffc);
 868         u32 show = stat;
 869         char msg[128];
 870
 871         if (stat & 0x00800000) {
 872                 if (!gk104_fifo_swmthd(fifo, chid, mthd, data))
 873                         show &= ~0x00800000;
 874                 nvkm_wr32(device, 0x0400c0 + (unit * 0x2000), 0x80600008);
 875         }
 876
 877         if (show) {
 878                 nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_0, show);
 879                 nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%s] subc %d "
 880                                    "mthd %04x data %08x\n",
 881                            unit, show, msg, chid,
 882                            nvkm_client_name_for_fifo_chid(&fifo->base, chid),
 883                            subc, mthd, data);
 884         }
 885
 886         nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat);
 887 }
 888
 889 static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = {
 890         { 0x00000001, "HCE_RE_ILLEGAL_OP" },
 891         { 0x00000002, "HCE_RE_ALIGNB" },
 892         { 0x00000004, "HCE_PRIV" },
 893         { 0x00000008, "HCE_ILLEGAL_MTHD" },
 894         { 0x00000010, "HCE_ILLEGAL_CLASS" },
 895         {}
 896 };
 897
 898 static void
 899 gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit)
 900 {
 901         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 902         struct nvkm_device *device = subdev->device;
 903         u32 mask = nvkm_rd32(device, 0x04014c + (unit * 0x2000));
 904         u32 stat = nvkm_rd32(device, 0x040148 + (unit * 0x2000)) & mask;
 905         u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff;
 906         char msg[128];
 907
 908         if (stat) {
 909                 nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_1, stat);
 910                 nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n",
 911                            unit, stat, msg, chid,
 912                            nvkm_rd32(device, 0x040150 + (unit * 0x2000)),
 913                            nvkm_rd32(device, 0x040154 + (unit * 0x2000)));
 914         }
 915
 916         nvkm_wr32(device, 0x040148 + (unit * 0x2000), stat);
 917 }
 918
 919 static void
 920 gk104_fifo_intr_runlist(struct gk104_fifo *fifo)
 921 {
 922         struct nvkm_device *device = fifo->base.engine.subdev.device;
 923         u32 mask = nvkm_rd32(device, 0x002a00);
 924         while (mask) {
 925                 u32 engn = __ffs(mask);
 926                 wake_up(&fifo->engine[engn].wait);
 927                 nvkm_wr32(device, 0x002a00, 1 << engn);
 928                 mask &= ~(1 << engn);
 929         }
 930 }
 931
 932 static void
 933 gk104_fifo_intr_engine(struct gk104_fifo *fifo)
 934 {
 935         nvkm_fifo_uevent(&fifo->base);
 936 }
 937
 938 static void
 939 gk104_fifo_intr(struct nvkm_subdev *subdev)
 940 {
 941         struct gk104_fifo *fifo = (void *)subdev;
 942         struct nvkm_device *device = fifo->base.engine.subdev.device;
 943         u32 mask = nvkm_rd32(device, 0x002140);
 944         u32 stat = nvkm_rd32(device, 0x002100) & mask;
 945
 946         if (stat & 0x00000001) {
 947                 gk104_fifo_intr_bind(fifo);
 948                 nvkm_wr32(device, 0x002100, 0x00000001);
 949                 stat &= ~0x00000001;
 950         }
 951
 952         if (stat & 0x00000010) {
 953                 nvkm_error(subdev, "PIO_ERROR\n");
 954                 nvkm_wr32(device, 0x002100, 0x00000010);
 955                 stat &= ~0x00000010;
 956         }
 957
 958         if (stat & 0x00000100) {
 959                 gk104_fifo_intr_sched(fifo);
 960                 nvkm_wr32(device, 0x002100, 0x00000100);
 961                 stat &= ~0x00000100;
 962         }
 963
 964         if (stat & 0x00010000) {
 965                 gk104_fifo_intr_chsw(fifo);
 966                 nvkm_wr32(device, 0x002100, 0x00010000);
 967                 stat &= ~0x00010000;
 968         }
 969
 970         if (stat & 0x00800000) {
 971                 nvkm_error(subdev, "FB_FLUSH_TIMEOUT\n");
 972                 nvkm_wr32(device, 0x002100, 0x00800000);
 973                 stat &= ~0x00800000;
 974         }
 975
 976         if (stat & 0x01000000) {
 977                 nvkm_error(subdev, "LB_ERROR\n");
 978                 nvkm_wr32(device, 0x002100, 0x01000000);
 979                 stat &= ~0x01000000;
 980         }
 981
 982         if (stat & 0x08000000) {
 983                 gk104_fifo_intr_dropped_fault(fifo);
 984                 nvkm_wr32(device, 0x002100, 0x08000000);
 985                 stat &= ~0x08000000;
 986         }
 987
 988         if (stat & 0x10000000) {
 989                 u32 mask = nvkm_rd32(device, 0x00259c);
 990                 while (mask) {
 991                         u32 unit = __ffs(mask);
 992                         gk104_fifo_intr_fault(fifo, unit);
 993                         nvkm_wr32(device, 0x00259c, (1 << unit));
 994                         mask &= ~(1 << unit);
 995                 }
 996                 stat &= ~0x10000000;
 997         }
 998
 999         if (stat & 0x20000000) {
1000                 u32 mask = nvkm_rd32(device, 0x0025a0);
1001                 while (mask) {
1002                         u32 unit = __ffs(mask);
1003                         gk104_fifo_intr_pbdma_0(fifo, unit);
1004                         gk104_fifo_intr_pbdma_1(fifo, unit);
1005                         nvkm_wr32(device, 0x0025a0, (1 << unit));
1006                         mask &= ~(1 << unit);
1007                 }
1008                 stat &= ~0x20000000;
1009         }
1010
1011         if (stat & 0x40000000) {
1012                 gk104_fifo_intr_runlist(fifo);
1013                 stat &= ~0x40000000;
1014         }
1015
1016         if (stat & 0x80000000) {
1017                 nvkm_wr32(device, 0x002100, 0x80000000);
1018                 gk104_fifo_intr_engine(fifo);
1019                 stat &= ~0x80000000;
1020         }
1021
1022         if (stat) {
1023                 nvkm_error(subdev, "INTR %08x\n", stat);
1024                 nvkm_mask(device, 0x002140, stat, 0x00000000);
1025                 nvkm_wr32(device, 0x002100, stat);
1026         }
1027 }
1028
1029 static void
1030 gk104_fifo_uevent_init(struct nvkm_event *event, int type, int index)
1031 {
1032         struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), uevent);
1033         struct nvkm_device *device = fifo->engine.subdev.device;
1034         nvkm_mask(device, 0x002140, 0x80000000, 0x80000000);
1035 }
1036
1037 static void
1038 gk104_fifo_uevent_fini(struct nvkm_event *event, int type, int index)
1039 {
1040         struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), uevent);
1041         struct nvkm_device *device = fifo->engine.subdev.device;
1042         nvkm_mask(device, 0x002140, 0x80000000, 0x00000000);
1043 }
1044
1045 static const struct nvkm_event_func
1046 gk104_fifo_uevent_func = {
1047         .ctor = nvkm_fifo_uevent_ctor,
1048         .init = gk104_fifo_uevent_init,
1049         .fini = gk104_fifo_uevent_fini,
1050 };
1051
1052 int
1053 gk104_fifo_fini(struct nvkm_object *object, bool suspend)
1054 {
1055         struct gk104_fifo *fifo = (void *)object;
1056         struct nvkm_device *device = fifo->base.engine.subdev.device;
1057         int ret;
1058
1059         ret = nvkm_fifo_fini(&fifo->base, suspend);
1060         if (ret)
1061                 return ret;
1062
1063         /* allow mmu fault interrupts, even when we're not using fifo */
1064         nvkm_mask(device, 0x002140, 0x10000000, 0x10000000);
1065         return 0;
1066 }
1067
1068 int
1069 gk104_fifo_init(struct nvkm_object *object)
1070 {
1071         struct gk104_fifo *fifo = (void *)object;
1072         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
1073         struct nvkm_device *device = subdev->device;
1074         int ret, i;
1075
1076         ret = nvkm_fifo_init(&fifo->base);
1077         if (ret)
1078                 return ret;
1079
1080         /* enable all available PBDMA units */
1081         nvkm_wr32(device, 0x000204, 0xffffffff);
1082         fifo->spoon_nr = hweight32(nvkm_rd32(device, 0x000204));
1083         nvkm_debug(subdev, "%d PBDMA unit(s)\n", fifo->spoon_nr);
1084
1085         /* PBDMA[n] */
1086         for (i = 0; i < fifo->spoon_nr; i++) {
1087                 nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
1088                 nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
1089                 nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
1090         }
1091
1092         /* PBDMA[n].HCE */
1093         for (i = 0; i < fifo->spoon_nr; i++) {
1094                 nvkm_wr32(device, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */
1095                 nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */
1096         }
1097
1098         nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar.offset >> 12);
1099
1100         nvkm_wr32(device, 0x002100, 0xffffffff);
1101         nvkm_wr32(device, 0x002140, 0x7fffffff);
1102         return 0;
1103 }
1104
1105 void
1106 gk104_fifo_dtor(struct nvkm_object *object)
1107 {
1108         struct gk104_fifo *fifo = (void *)object;
1109         int i;
1110
1111         nvkm_gpuobj_unmap(&fifo->user.bar);
1112         nvkm_gpuobj_ref(NULL, &fifo->user.mem);
1113
1114         for (i = 0; i < FIFO_ENGINE_NR; i++) {
1115                 nvkm_gpuobj_ref(NULL, &fifo->engine[i].runlist[1]);
1116                 nvkm_gpuobj_ref(NULL, &fifo->engine[i].runlist[0]);
1117         }
1118
1119         nvkm_fifo_destroy(&fifo->base);
1120 }
1121
1122 int
1123 gk104_fifo_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
1124                 struct nvkm_oclass *oclass, void *data, u32 size,
1125                 struct nvkm_object **pobject)
1126 {
1127         struct gk104_fifo_impl *impl = (void *)oclass;
1128         struct gk104_fifo *fifo;
1129         int ret, i;
1130
1131         ret = nvkm_fifo_create(parent, engine, oclass, 0,
1132                                impl->channels - 1, &fifo);
1133         *pobject = nv_object(fifo);
1134         if (ret)
1135                 return ret;
1136
1137         INIT_WORK(&fifo->fault, gk104_fifo_recover_work);
1138
1139         for (i = 0; i < FIFO_ENGINE_NR; i++) {
1140                 ret = nvkm_gpuobj_new(nv_object(fifo), NULL, 0x8000, 0x1000,
1141                                       0, &fifo->engine[i].runlist[0]);
1142                 if (ret)
1143                         return ret;
1144
1145                 ret = nvkm_gpuobj_new(nv_object(fifo), NULL, 0x8000, 0x1000,
1146                                       0, &fifo->engine[i].runlist[1]);
1147                 if (ret)
1148                         return ret;
1149
1150                 init_waitqueue_head(&fifo->engine[i].wait);
1151         }
1152
1153         ret = nvkm_gpuobj_new(nv_object(fifo), NULL, impl->channels * 0x200,
1154                               0x1000, NVOBJ_FLAG_ZERO_ALLOC, &fifo->user.mem);
1155         if (ret)
1156                 return ret;
1157
1158         ret = nvkm_gpuobj_map(fifo->user.mem, NV_MEM_ACCESS_RW,
1159                               &fifo->user.bar);
1160         if (ret)
1161                 return ret;
1162
1163         ret = nvkm_event_init(&gk104_fifo_uevent_func, 1, 1, &fifo->base.uevent);
1164         if (ret)
1165                 return ret;
1166
1167         nv_subdev(fifo)->unit = 0x00000100;
1168         nv_subdev(fifo)->intr = gk104_fifo_intr;
1169         nv_engine(fifo)->cclass = &gk104_fifo_cclass;
1170         nv_engine(fifo)->sclass = gk104_fifo_sclass;
1171         return 0;
1172 }
1173
1174 struct nvkm_oclass *
1175 gk104_fifo_oclass = &(struct gk104_fifo_impl) {
1176         .base.handle = NV_ENGINE(FIFO, 0xe0),
1177         .base.ofuncs = &(struct nvkm_ofuncs) {
1178                 .ctor = gk104_fifo_ctor,
1179                 .dtor = gk104_fifo_dtor,
1180                 .init = gk104_fifo_init,
1181                 .fini = gk104_fifo_fini,
1182         },
1183         .channels = 4096,
1184 }.base;