drivers/gpu/drm/nouveau/nvc0_graph.c

   1 /*
   2  * Copyright 2010 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include <linux/firmware.h>
  26
  27 #include "drmP.h"
  28
  29 #include "nouveau_drv.h"
  30 #include "nouveau_mm.h"
  31
  32 #include "nvc0_graph.h"
  33 #include "nvc0_grhub.fuc.h"
  34 #include "nvc0_grgpc.fuc.h"
  35
  36 static void
  37 nvc0_graph_ctxctl_debug_unit(struct drm_device *dev, u32 base)
  38 {
  39         NV_INFO(dev, "PGRAPH: %06x - done 0x%08x\n", base,
  40                 nv_rd32(dev, base + 0x400));
  41         NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
  42                 nv_rd32(dev, base + 0x800), nv_rd32(dev, base + 0x804),
  43                 nv_rd32(dev, base + 0x808), nv_rd32(dev, base + 0x80c));
  44         NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
  45                 nv_rd32(dev, base + 0x810), nv_rd32(dev, base + 0x814),
  46                 nv_rd32(dev, base + 0x818), nv_rd32(dev, base + 0x81c));
  47 }
  48
  49 static void
  50 nvc0_graph_ctxctl_debug(struct drm_device *dev)
  51 {
  52         u32 gpcnr = nv_rd32(dev, 0x409604) & 0xffff;
  53         u32 gpc;
  54
  55         nvc0_graph_ctxctl_debug_unit(dev, 0x409000);
  56         for (gpc = 0; gpc < gpcnr; gpc++)
  57                 nvc0_graph_ctxctl_debug_unit(dev, 0x502000 + (gpc * 0x8000));
  58 }
  59
  60 static int
  61 nvc0_graph_load_context(struct nouveau_channel *chan)
  62 {
  63         struct drm_device *dev = chan->dev;
  64
  65         nv_wr32(dev, 0x409840, 0x00000030);
  66         nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
  67         nv_wr32(dev, 0x409504, 0x00000003);
  68         if (!nv_wait(dev, 0x409800, 0x00000010, 0x00000010))
  69                 NV_ERROR(dev, "PGRAPH: load_ctx timeout\n");
  70
  71         return 0;
  72 }
  73
  74 static int
  75 nvc0_graph_unload_context_to(struct drm_device *dev, u64 chan)
  76 {
  77         nv_wr32(dev, 0x409840, 0x00000003);
  78         nv_wr32(dev, 0x409500, 0x80000000 | chan >> 12);
  79         nv_wr32(dev, 0x409504, 0x00000009);
  80         if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000000)) {
  81                 NV_ERROR(dev, "PGRAPH: unload_ctx timeout\n");
  82                 return -EBUSY;
  83         }
  84
  85         return 0;
  86 }
  87
  88 static int
  89 nvc0_graph_construct_context(struct nouveau_channel *chan)
  90 {
  91         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
  92         struct nvc0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
  93         struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
  94         struct drm_device *dev = chan->dev;
  95         int ret, i;
  96         u32 *ctx;
  97
  98         ctx = kmalloc(priv->grctx_size, GFP_KERNEL);
  99         if (!ctx)
 100                 return -ENOMEM;
 101
 102         if (!nouveau_ctxfw) {
 103                 nv_wr32(dev, 0x409840, 0x80000000);
 104                 nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
 105                 nv_wr32(dev, 0x409504, 0x00000001);
 106                 if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
 107                         NV_ERROR(dev, "PGRAPH: HUB_SET_CHAN timeout\n");
 108                         nvc0_graph_ctxctl_debug(dev);
 109                         return -EBUSY;
 110                 }
 111         } else {
 112                 nvc0_graph_load_context(chan);
 113
 114                 nv_wo32(grch->grctx, 0x1c, 1);
 115                 nv_wo32(grch->grctx, 0x20, 0);
 116                 nv_wo32(grch->grctx, 0x28, 0);
 117                 nv_wo32(grch->grctx, 0x2c, 0);
 118                 dev_priv->engine.instmem.flush(dev);
 119         }
 120
 121         ret = nvc0_grctx_generate(chan);
 122         if (ret) {
 123                 kfree(ctx);
 124                 return ret;
 125         }
 126
 127         if (!nouveau_ctxfw) {
 128                 nv_wr32(dev, 0x409840, 0x80000000);
 129                 nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
 130                 nv_wr32(dev, 0x409504, 0x00000002);
 131                 if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
 132                         NV_ERROR(dev, "PGRAPH: HUB_CTX_SAVE timeout\n");
 133                         nvc0_graph_ctxctl_debug(dev);
 134                         return -EBUSY;
 135                 }
 136         } else {
 137                 ret = nvc0_graph_unload_context_to(dev, chan->ramin->vinst);
 138                 if (ret) {
 139                         kfree(ctx);
 140                         return ret;
 141                 }
 142         }
 143
 144         for (i = 0; i < priv->grctx_size; i += 4)
 145                 ctx[i / 4] = nv_ro32(grch->grctx, i);
 146
 147         priv->grctx_vals = ctx;
 148         return 0;
 149 }
 150
 151 static int
 152 nvc0_graph_create_context_mmio_list(struct nouveau_channel *chan)
 153 {
 154         struct nvc0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
 155         struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
 156         struct drm_device *dev = chan->dev;
 157         int i = 0, gpc, tp, ret;
 158         u32 magic;
 159
 160         ret = nouveau_gpuobj_new(dev, NULL, 0x2000, 256, NVOBJ_FLAG_VM,
 161                                  &grch->unk408004);
 162         if (ret)
 163                 return ret;
 164
 165         ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 256, NVOBJ_FLAG_VM,
 166                                  &grch->unk40800c);
 167         if (ret)
 168                 return ret;
 169
 170         ret = nouveau_gpuobj_new(dev, NULL, 384 * 1024, 4096,
 171                                  NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER,
 172                                  &grch->unk418810);
 173         if (ret)
 174                 return ret;
 175
 176         ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0, NVOBJ_FLAG_VM,
 177                                  &grch->mmio);
 178         if (ret)
 179                 return ret;
 180
 181
 182         nv_wo32(grch->mmio, i++ * 4, 0x00408004);
 183         nv_wo32(grch->mmio, i++ * 4, grch->unk408004->linst >> 8);
 184         nv_wo32(grch->mmio, i++ * 4, 0x00408008);
 185         nv_wo32(grch->mmio, i++ * 4, 0x80000018);
 186
 187         nv_wo32(grch->mmio, i++ * 4, 0x0040800c);
 188         nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->linst >> 8);
 189         nv_wo32(grch->mmio, i++ * 4, 0x00408010);
 190         nv_wo32(grch->mmio, i++ * 4, 0x80000000);
 191
 192         nv_wo32(grch->mmio, i++ * 4, 0x00418810);
 193         nv_wo32(grch->mmio, i++ * 4, 0x80000000 | grch->unk418810->linst >> 12);
 194         nv_wo32(grch->mmio, i++ * 4, 0x00419848);
 195         nv_wo32(grch->mmio, i++ * 4, 0x10000000 | grch->unk418810->linst >> 12);
 196
 197         nv_wo32(grch->mmio, i++ * 4, 0x00419004);
 198         nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->linst >> 8);
 199         nv_wo32(grch->mmio, i++ * 4, 0x00419008);
 200         nv_wo32(grch->mmio, i++ * 4, 0x00000000);
 201
 202         nv_wo32(grch->mmio, i++ * 4, 0x00418808);
 203         nv_wo32(grch->mmio, i++ * 4, grch->unk408004->linst >> 8);
 204         nv_wo32(grch->mmio, i++ * 4, 0x0041880c);
 205         nv_wo32(grch->mmio, i++ * 4, 0x80000018);
 206
 207         magic = 0x02180000;
 208         nv_wo32(grch->mmio, i++ * 4, 0x00405830);
 209         nv_wo32(grch->mmio, i++ * 4, magic);
 210         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 211                 for (tp = 0; tp < priv->tp_nr[gpc]; tp++, magic += 0x0324) {
 212                         u32 reg = 0x504520 + (gpc * 0x8000) + (tp * 0x0800);
 213                         nv_wo32(grch->mmio, i++ * 4, reg);
 214                         nv_wo32(grch->mmio, i++ * 4, magic);
 215                 }
 216         }
 217
 218         grch->mmio_nr = i / 2;
 219         return 0;
 220 }
 221
 222 static int
 223 nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
 224 {
 225         struct drm_device *dev = chan->dev;
 226         struct drm_nouveau_private *dev_priv = dev->dev_private;
 227         struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
 228         struct nvc0_graph_priv *priv = nv_engine(dev, engine);
 229         struct nvc0_graph_chan *grch;
 230         struct nouveau_gpuobj *grctx;
 231         int ret, i;
 232
 233         grch = kzalloc(sizeof(*grch), GFP_KERNEL);
 234         if (!grch)
 235                 return -ENOMEM;
 236         chan->engctx[NVOBJ_ENGINE_GR] = grch;
 237
 238         ret = nouveau_gpuobj_new(dev, NULL, priv->grctx_size, 256,
 239                                  NVOBJ_FLAG_VM | NVOBJ_FLAG_ZERO_ALLOC,
 240                                  &grch->grctx);
 241         if (ret)
 242                 goto error;
 243         grctx = grch->grctx;
 244
 245         ret = nvc0_graph_create_context_mmio_list(chan);
 246         if (ret)
 247                 goto error;
 248
 249         nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->linst) | 4);
 250         nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->linst));
 251         pinstmem->flush(dev);
 252
 253         if (!priv->grctx_vals) {
 254                 ret = nvc0_graph_construct_context(chan);
 255                 if (ret)
 256                         goto error;
 257         }
 258
 259         for (i = 0; i < priv->grctx_size; i += 4)
 260                 nv_wo32(grctx, i, priv->grctx_vals[i / 4]);
 261
 262         if (!nouveau_ctxfw) {
 263                 nv_wo32(grctx, 0x00, grch->mmio_nr);
 264                 nv_wo32(grctx, 0x04, grch->mmio->linst >> 8);
 265         } else {
 266                 nv_wo32(grctx, 0xf4, 0);
 267                 nv_wo32(grctx, 0xf8, 0);
 268                 nv_wo32(grctx, 0x10, grch->mmio_nr);
 269                 nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->linst));
 270                 nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->linst));
 271                 nv_wo32(grctx, 0x1c, 1);
 272                 nv_wo32(grctx, 0x20, 0);
 273                 nv_wo32(grctx, 0x28, 0);
 274                 nv_wo32(grctx, 0x2c, 0);
 275         }
 276         pinstmem->flush(dev);
 277         return 0;
 278
 279 error:
 280         priv->base.context_del(chan, engine);
 281         return ret;
 282 }
 283
 284 static void
 285 nvc0_graph_context_del(struct nouveau_channel *chan, int engine)
 286 {
 287         struct nvc0_graph_chan *grch = chan->engctx[engine];
 288
 289         nouveau_gpuobj_ref(NULL, &grch->mmio);
 290         nouveau_gpuobj_ref(NULL, &grch->unk418810);
 291         nouveau_gpuobj_ref(NULL, &grch->unk40800c);
 292         nouveau_gpuobj_ref(NULL, &grch->unk408004);
 293         nouveau_gpuobj_ref(NULL, &grch->grctx);
 294         chan->engctx[engine] = NULL;
 295 }
 296
 297 static int
 298 nvc0_graph_object_new(struct nouveau_channel *chan, int engine,
 299                       u32 handle, u16 class)
 300 {
 301         return 0;
 302 }
 303
 304 static int
 305 nvc0_graph_fini(struct drm_device *dev, int engine)
 306 {
 307         return 0;
 308 }
 309
 310 static int
 311 nvc0_graph_mthd_page_flip(struct nouveau_channel *chan,
 312                           u32 class, u32 mthd, u32 data)
 313 {
 314         nouveau_finish_page_flip(chan, NULL);
 315         return 0;
 316 }
 317
 318 static void
 319 nvc0_graph_init_obj418880(struct drm_device *dev)
 320 {
 321         struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 322         int i;
 323
 324         nv_wr32(dev, GPC_BCAST(0x0880), 0x00000000);
 325         nv_wr32(dev, GPC_BCAST(0x08a4), 0x00000000);
 326         for (i = 0; i < 4; i++)
 327                 nv_wr32(dev, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
 328         nv_wr32(dev, GPC_BCAST(0x08b4), priv->unk4188b4->vinst >> 8);
 329         nv_wr32(dev, GPC_BCAST(0x08b8), priv->unk4188b8->vinst >> 8);
 330 }
 331
 332 static void
 333 nvc0_graph_init_regs(struct drm_device *dev)
 334 {
 335         nv_wr32(dev, 0x400080, 0x003083c2);
 336         nv_wr32(dev, 0x400088, 0x00006fe7);
 337         nv_wr32(dev, 0x40008c, 0x00000000);
 338         nv_wr32(dev, 0x400090, 0x00000030);
 339         nv_wr32(dev, 0x40013c, 0x013901f7);
 340         nv_wr32(dev, 0x400140, 0x00000100);
 341         nv_wr32(dev, 0x400144, 0x00000000);
 342         nv_wr32(dev, 0x400148, 0x00000110);
 343         nv_wr32(dev, 0x400138, 0x00000000);
 344         nv_wr32(dev, 0x400130, 0x00000000);
 345         nv_wr32(dev, 0x400134, 0x00000000);
 346         nv_wr32(dev, 0x400124, 0x00000002);
 347 }
 348
 349 static void
 350 nvc0_graph_init_gpc_0(struct drm_device *dev)
 351 {
 352         struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 353         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tp_total);
 354         u32 data[TP_MAX / 8];
 355         u8  tpnr[GPC_MAX];
 356         int i, gpc, tpc;
 357
 358         /*
 359          *      TP      ROP UNKVAL(magic_not_rop_nr)
 360          * 450: 4/0/0/0 2        3
 361          * 460: 3/4/0/0 4        1
 362          * 465: 3/4/4/0 4        7
 363          * 470: 3/3/4/4 5        5
 364          * 480: 3/4/4/4 6        6
 365          */
 366
 367         memset(data, 0x00, sizeof(data));
 368         memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
 369         for (i = 0, gpc = -1; i < priv->tp_total; i++) {
 370                 do {
 371                         gpc = (gpc + 1) % priv->gpc_nr;
 372                 } while (!tpnr[gpc]);
 373                 tpc = priv->tp_nr[gpc] - tpnr[gpc]--;
 374
 375                 data[i / 8] |= tpc << ((i % 8) * 4);
 376         }
 377
 378         nv_wr32(dev, GPC_BCAST(0x0980), data[0]);
 379         nv_wr32(dev, GPC_BCAST(0x0984), data[1]);
 380         nv_wr32(dev, GPC_BCAST(0x0988), data[2]);
 381         nv_wr32(dev, GPC_BCAST(0x098c), data[3]);
 382
 383         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 384                 nv_wr32(dev, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
 385                                                   priv->tp_nr[gpc]);
 386                 nv_wr32(dev, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tp_total);
 387                 nv_wr32(dev, GPC_UNIT(gpc, 0x0918), magicgpc918);
 388         }
 389
 390         nv_wr32(dev, GPC_BCAST(0x1bd4), magicgpc918);
 391         nv_wr32(dev, GPC_BCAST(0x08ac), priv->rop_nr);
 392 }
 393
 394 static void
 395 nvc0_graph_init_units(struct drm_device *dev)
 396 {
 397         nv_wr32(dev, 0x409c24, 0x000f0000);
 398         nv_wr32(dev, 0x404000, 0xc0000000); /* DISPATCH */
 399         nv_wr32(dev, 0x404600, 0xc0000000); /* M2MF */
 400         nv_wr32(dev, 0x408030, 0xc0000000);
 401         nv_wr32(dev, 0x40601c, 0xc0000000);
 402         nv_wr32(dev, 0x404490, 0xc0000000); /* MACRO */
 403         nv_wr32(dev, 0x406018, 0xc0000000);
 404         nv_wr32(dev, 0x405840, 0xc0000000);
 405         nv_wr32(dev, 0x405844, 0x00ffffff);
 406         nv_mask(dev, 0x419cc0, 0x00000008, 0x00000008);
 407         nv_mask(dev, 0x419eb4, 0x00001000, 0x00001000);
 408 }
 409
 410 static void
 411 nvc0_graph_init_gpc_1(struct drm_device *dev)
 412 {
 413         struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 414         int gpc, tp;
 415
 416         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 417                 nv_wr32(dev, GPC_UNIT(gpc, 0x0420), 0xc0000000);
 418                 nv_wr32(dev, GPC_UNIT(gpc, 0x0900), 0xc0000000);
 419                 nv_wr32(dev, GPC_UNIT(gpc, 0x1028), 0xc0000000);
 420                 nv_wr32(dev, GPC_UNIT(gpc, 0x0824), 0xc0000000);
 421                 for (tp = 0; tp < priv->tp_nr[gpc]; tp++) {
 422                         nv_wr32(dev, TP_UNIT(gpc, tp, 0x508), 0xffffffff);
 423                         nv_wr32(dev, TP_UNIT(gpc, tp, 0x50c), 0xffffffff);
 424                         nv_wr32(dev, TP_UNIT(gpc, tp, 0x224), 0xc0000000);
 425                         nv_wr32(dev, TP_UNIT(gpc, tp, 0x48c), 0xc0000000);
 426                         nv_wr32(dev, TP_UNIT(gpc, tp, 0x084), 0xc0000000);
 427                         nv_wr32(dev, TP_UNIT(gpc, tp, 0x644), 0x001ffffe);
 428                         nv_wr32(dev, TP_UNIT(gpc, tp, 0x64c), 0x0000000f);
 429                 }
 430                 nv_wr32(dev, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
 431                 nv_wr32(dev, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
 432         }
 433 }
 434
 435 static void
 436 nvc0_graph_init_rop(struct drm_device *dev)
 437 {
 438         struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 439         int rop;
 440
 441         for (rop = 0; rop < priv->rop_nr; rop++) {
 442                 nv_wr32(dev, ROP_UNIT(rop, 0x144), 0xc0000000);
 443                 nv_wr32(dev, ROP_UNIT(rop, 0x070), 0xc0000000);
 444                 nv_wr32(dev, ROP_UNIT(rop, 0x204), 0xffffffff);
 445                 nv_wr32(dev, ROP_UNIT(rop, 0x208), 0xffffffff);
 446         }
 447 }
 448
 449 static void
 450 nvc0_graph_init_fuc(struct drm_device *dev, u32 fuc_base,
 451                     struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
 452 {
 453         int i;
 454
 455         nv_wr32(dev, fuc_base + 0x01c0, 0x01000000);
 456         for (i = 0; i < data->size / 4; i++)
 457                 nv_wr32(dev, fuc_base + 0x01c4, data->data[i]);
 458
 459         nv_wr32(dev, fuc_base + 0x0180, 0x01000000);
 460         for (i = 0; i < code->size / 4; i++) {
 461                 if ((i & 0x3f) == 0)
 462                         nv_wr32(dev, fuc_base + 0x0188, i >> 6);
 463                 nv_wr32(dev, fuc_base + 0x0184, code->data[i]);
 464         }
 465 }
 466
 467 static int
 468 nvc0_graph_init_ctxctl(struct drm_device *dev)
 469 {
 470         struct drm_nouveau_private *dev_priv = dev->dev_private;
 471         struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
 472         u32 r000260;
 473         int i;
 474
 475         if (!nouveau_ctxfw) {
 476                 /* load HUB microcode */
 477                 r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
 478                 nv_wr32(dev, 0x4091c0, 0x01000000);
 479                 for (i = 0; i < sizeof(nvc0_grhub_data) / 4; i++)
 480                         nv_wr32(dev, 0x4091c4, nvc0_grhub_data[i]);
 481
 482                 nv_wr32(dev, 0x409180, 0x01000000);
 483                 for (i = 0; i < sizeof(nvc0_grhub_code) / 4; i++) {
 484                         if ((i & 0x3f) == 0)
 485                                 nv_wr32(dev, 0x409188, i >> 6);
 486                         nv_wr32(dev, 0x409184, nvc0_grhub_code[i]);
 487                 }
 488
 489                 /* load GPC microcode */
 490                 nv_wr32(dev, 0x41a1c0, 0x01000000);
 491                 for (i = 0; i < sizeof(nvc0_grgpc_data) / 4; i++)
 492                         nv_wr32(dev, 0x41a1c4, nvc0_grgpc_data[i]);
 493
 494                 nv_wr32(dev, 0x41a180, 0x01000000);
 495                 for (i = 0; i < sizeof(nvc0_grgpc_code) / 4; i++) {
 496                         if ((i & 0x3f) == 0)
 497                                 nv_wr32(dev, 0x41a188, i >> 6);
 498                         nv_wr32(dev, 0x41a184, nvc0_grgpc_code[i]);
 499                 }
 500                 nv_wr32(dev, 0x000260, r000260);
 501
 502                 /* start HUB ucode running, it'll init the GPCs */
 503                 nv_wr32(dev, 0x409800, dev_priv->chipset);
 504                 nv_wr32(dev, 0x40910c, 0x00000000);
 505                 nv_wr32(dev, 0x409100, 0x00000002);
 506                 if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
 507                         NV_ERROR(dev, "PGRAPH: HUB_INIT timed out\n");
 508                         nvc0_graph_ctxctl_debug(dev);
 509                         return -EBUSY;
 510                 }
 511
 512                 priv->grctx_size = nv_rd32(dev, 0x409804);
 513                 return 0;
 514         }
 515
 516         /* load fuc microcode */
 517         r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
 518         nvc0_graph_init_fuc(dev, 0x409000, &priv->fuc409c, &priv->fuc409d);
 519         nvc0_graph_init_fuc(dev, 0x41a000, &priv->fuc41ac, &priv->fuc41ad);
 520         nv_wr32(dev, 0x000260, r000260);
 521
 522         /* start both of them running */
 523         nv_wr32(dev, 0x409840, 0xffffffff);
 524         nv_wr32(dev, 0x41a10c, 0x00000000);
 525         nv_wr32(dev, 0x40910c, 0x00000000);
 526         nv_wr32(dev, 0x41a100, 0x00000002);
 527         nv_wr32(dev, 0x409100, 0x00000002);
 528         if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000001))
 529                 NV_INFO(dev, "0x409800 wait failed\n");
 530
 531         nv_wr32(dev, 0x409840, 0xffffffff);
 532         nv_wr32(dev, 0x409500, 0x7fffffff);
 533         nv_wr32(dev, 0x409504, 0x00000021);
 534
 535         nv_wr32(dev, 0x409840, 0xffffffff);
 536         nv_wr32(dev, 0x409500, 0x00000000);
 537         nv_wr32(dev, 0x409504, 0x00000010);
 538         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 539                 NV_ERROR(dev, "fuc09 req 0x10 timeout\n");
 540                 return -EBUSY;
 541         }
 542         priv->grctx_size = nv_rd32(dev, 0x409800);
 543
 544         nv_wr32(dev, 0x409840, 0xffffffff);
 545         nv_wr32(dev, 0x409500, 0x00000000);
 546         nv_wr32(dev, 0x409504, 0x00000016);
 547         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 548                 NV_ERROR(dev, "fuc09 req 0x16 timeout\n");
 549                 return -EBUSY;
 550         }
 551
 552         nv_wr32(dev, 0x409840, 0xffffffff);
 553         nv_wr32(dev, 0x409500, 0x00000000);
 554         nv_wr32(dev, 0x409504, 0x00000025);
 555         if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
 556                 NV_ERROR(dev, "fuc09 req 0x25 timeout\n");
 557                 return -EBUSY;
 558         }
 559
 560         return 0;
 561 }
 562
 563 static int
 564 nvc0_graph_init(struct drm_device *dev, int engine)
 565 {
 566         int ret;
 567
 568         nv_mask(dev, 0x000200, 0x18001000, 0x00000000);
 569         nv_mask(dev, 0x000200, 0x18001000, 0x18001000);
 570
 571         nvc0_graph_init_obj418880(dev);
 572         nvc0_graph_init_regs(dev);
 573         /*nvc0_graph_init_unitplemented_magics(dev);*/
 574         nvc0_graph_init_gpc_0(dev);
 575         /*nvc0_graph_init_unitplemented_c242(dev);*/
 576
 577         nv_wr32(dev, 0x400500, 0x00010001);
 578         nv_wr32(dev, 0x400100, 0xffffffff);
 579         nv_wr32(dev, 0x40013c, 0xffffffff);
 580
 581         nvc0_graph_init_units(dev);
 582         nvc0_graph_init_gpc_1(dev);
 583         nvc0_graph_init_rop(dev);
 584
 585         nv_wr32(dev, 0x400108, 0xffffffff);
 586         nv_wr32(dev, 0x400138, 0xffffffff);
 587         nv_wr32(dev, 0x400118, 0xffffffff);
 588         nv_wr32(dev, 0x400130, 0xffffffff);
 589         nv_wr32(dev, 0x40011c, 0xffffffff);
 590         nv_wr32(dev, 0x400134, 0xffffffff);
 591         nv_wr32(dev, 0x400054, 0x34ce3464);
 592
 593         ret = nvc0_graph_init_ctxctl(dev);
 594         if (ret)
 595                 return ret;
 596
 597         return 0;
 598 }
 599
 600 int
 601 nvc0_graph_isr_chid(struct drm_device *dev, u64 inst)
 602 {
 603         struct drm_nouveau_private *dev_priv = dev->dev_private;
 604         struct nouveau_channel *chan;
 605         unsigned long flags;
 606         int i;
 607
 608         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 609         for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 610                 chan = dev_priv->channels.ptr[i];
 611                 if (!chan || !chan->ramin)
 612                         continue;
 613
 614                 if (inst == chan->ramin->vinst)
 615                         break;
 616         }
 617         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 618         return i;
 619 }
 620
 621 static void
 622 nvc0_graph_ctxctl_isr(struct drm_device *dev)
 623 {
 624         u32 ustat = nv_rd32(dev, 0x409c18);
 625
 626         if (ustat & 0x00000001)
 627                 NV_INFO(dev, "PGRAPH: CTXCTRL ucode error\n");
 628         if (ustat & 0x00080000)
 629                 NV_INFO(dev, "PGRAPH: CTXCTRL watchdog timeout\n");
 630         if (ustat & ~0x00080001)
 631                 NV_INFO(dev, "PGRAPH: CTXCTRL 0x%08x\n", ustat);
 632
 633         nvc0_graph_ctxctl_debug(dev);
 634         nv_wr32(dev, 0x409c20, ustat);
 635 }
 636
 637 static void
 638 nvc0_graph_isr(struct drm_device *dev)
 639 {
 640         u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
 641         u32 chid = nvc0_graph_isr_chid(dev, inst);
 642         u32 stat = nv_rd32(dev, 0x400100);
 643         u32 addr = nv_rd32(dev, 0x400704);
 644         u32 mthd = (addr & 0x00003ffc);
 645         u32 subc = (addr & 0x00070000) >> 16;
 646         u32 data = nv_rd32(dev, 0x400708);
 647         u32 code = nv_rd32(dev, 0x400110);
 648         u32 class = nv_rd32(dev, 0x404200 + (subc * 4));
 649
 650         if (stat & 0x00000010) {
 651                 if (nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data)) {
 652                         NV_INFO(dev, "PGRAPH: ILLEGAL_MTHD ch %d [0x%010llx] "
 653                                      "subc %d class 0x%04x mthd 0x%04x "
 654                                      "data 0x%08x\n",
 655                                 chid, inst, subc, class, mthd, data);
 656                 }
 657                 nv_wr32(dev, 0x400100, 0x00000010);
 658                 stat &= ~0x00000010;
 659         }
 660
 661         if (stat & 0x00000020) {
 662                 NV_INFO(dev, "PGRAPH: ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
 663                              "class 0x%04x mthd 0x%04x data 0x%08x\n",
 664                         chid, inst, subc, class, mthd, data);
 665                 nv_wr32(dev, 0x400100, 0x00000020);
 666                 stat &= ~0x00000020;
 667         }
 668
 669         if (stat & 0x00100000) {
 670                 NV_INFO(dev, "PGRAPH: DATA_ERROR [");
 671                 nouveau_enum_print(nv50_data_error_names, code);
 672                 printk("] ch %d [0x%010llx] subc %d class 0x%04x "
 673                        "mthd 0x%04x data 0x%08x\n",
 674                        chid, inst, subc, class, mthd, data);
 675                 nv_wr32(dev, 0x400100, 0x00100000);
 676                 stat &= ~0x00100000;
 677         }
 678
 679         if (stat & 0x00200000) {
 680                 u32 trap = nv_rd32(dev, 0x400108);
 681                 NV_INFO(dev, "PGRAPH: TRAP ch %d status 0x%08x\n", chid, trap);
 682                 nv_wr32(dev, 0x400108, trap);
 683                 nv_wr32(dev, 0x400100, 0x00200000);
 684                 stat &= ~0x00200000;
 685         }
 686
 687         if (stat & 0x00080000) {
 688                 nvc0_graph_ctxctl_isr(dev);
 689                 nv_wr32(dev, 0x400100, 0x00080000);
 690                 stat &= ~0x00080000;
 691         }
 692
 693         if (stat) {
 694                 NV_INFO(dev, "PGRAPH: unknown stat 0x%08x\n", stat);
 695                 nv_wr32(dev, 0x400100, stat);
 696         }
 697
 698         nv_wr32(dev, 0x400500, 0x00010001);
 699 }
 700
 701 static void
 702 nvc0_runk140_isr(struct drm_device *dev)
 703 {
 704         u32 units = nv_rd32(dev, 0x00017c) & 0x1f;
 705
 706         while (units) {
 707                 u32 unit = ffs(units) - 1;
 708                 u32 reg = 0x140000 + unit * 0x2000;
 709                 u32 st0 = nv_mask(dev, reg + 0x1020, 0, 0);
 710                 u32 st1 = nv_mask(dev, reg + 0x1420, 0, 0);
 711
 712                 NV_INFO(dev, "PRUNK140: %d 0x%08x 0x%08x\n", unit, st0, st1);
 713                 units &= ~(1 << unit);
 714         }
 715 }
 716
 717 static int
 718 nvc0_graph_create_fw(struct drm_device *dev, const char *fwname,
 719                      struct nvc0_graph_fuc *fuc)
 720 {
 721         struct drm_nouveau_private *dev_priv = dev->dev_private;
 722         const struct firmware *fw;
 723         char f[32];
 724         int ret;
 725
 726         snprintf(f, sizeof(f), "nouveau/nv%02x_%s", dev_priv->chipset, fwname);
 727         ret = request_firmware(&fw, f, &dev->pdev->dev);
 728         if (ret) {
 729                 snprintf(f, sizeof(f), "nouveau/%s", fwname);
 730                 ret = request_firmware(&fw, f, &dev->pdev->dev);
 731                 if (ret) {
 732                         NV_ERROR(dev, "failed to load %s\n", fwname);
 733                         return ret;
 734                 }
 735         }
 736
 737         fuc->size = fw->size;
 738         fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
 739         release_firmware(fw);
 740         return (fuc->data != NULL) ? 0 : -ENOMEM;
 741 }
 742
 743 static void
 744 nvc0_graph_destroy_fw(struct nvc0_graph_fuc *fuc)
 745 {
 746         if (fuc->data) {
 747                 kfree(fuc->data);
 748                 fuc->data = NULL;
 749         }
 750 }
 751
 752 static void
 753 nvc0_graph_destroy(struct drm_device *dev, int engine)
 754 {
 755         struct nvc0_graph_priv *priv = nv_engine(dev, engine);
 756
 757         if (nouveau_ctxfw) {
 758                 nvc0_graph_destroy_fw(&priv->fuc409c);
 759                 nvc0_graph_destroy_fw(&priv->fuc409d);
 760                 nvc0_graph_destroy_fw(&priv->fuc41ac);
 761                 nvc0_graph_destroy_fw(&priv->fuc41ad);
 762         }
 763
 764         nouveau_irq_unregister(dev, 12);
 765         nouveau_irq_unregister(dev, 25);
 766
 767         nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
 768         nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
 769
 770         if (priv->grctx_vals)
 771                 kfree(priv->grctx_vals);
 772
 773         NVOBJ_ENGINE_DEL(dev, GR);
 774         kfree(priv);
 775 }
 776
 777 int
 778 nvc0_graph_create(struct drm_device *dev)
 779 {
 780         struct drm_nouveau_private *dev_priv = dev->dev_private;
 781         struct nvc0_graph_priv *priv;
 782         int ret, gpc, i;
 783         u32 fermi;
 784
 785         fermi = nvc0_graph_class(dev);
 786         if (!fermi) {
 787                 NV_ERROR(dev, "PGRAPH: unsupported chipset, please report!\n");
 788                 return 0;
 789         }
 790
 791         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 792         if (!priv)
 793                 return -ENOMEM;
 794
 795         priv->base.destroy = nvc0_graph_destroy;
 796         priv->base.init = nvc0_graph_init;
 797         priv->base.fini = nvc0_graph_fini;
 798         priv->base.context_new = nvc0_graph_context_new;
 799         priv->base.context_del = nvc0_graph_context_del;
 800         priv->base.object_new = nvc0_graph_object_new;
 801
 802         NVOBJ_ENGINE_ADD(dev, GR, &priv->base);
 803         nouveau_irq_register(dev, 12, nvc0_graph_isr);
 804         nouveau_irq_register(dev, 25, nvc0_runk140_isr);
 805
 806         if (nouveau_ctxfw) {
 807                 NV_INFO(dev, "PGRAPH: using external firmware\n");
 808                 if (nvc0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
 809                     nvc0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
 810                     nvc0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
 811                     nvc0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
 812                         ret = 0;
 813                         goto error;
 814                 }
 815         }
 816
 817         ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b4);
 818         if (ret)
 819                 goto error;
 820
 821         ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b8);
 822         if (ret)
 823                 goto error;
 824
 825         for (i = 0; i < 0x1000; i += 4) {
 826                 nv_wo32(priv->unk4188b4, i, 0x00000010);
 827                 nv_wo32(priv->unk4188b8, i, 0x00000010);
 828         }
 829
 830         priv->gpc_nr  =  nv_rd32(dev, 0x409604) & 0x0000001f;
 831         priv->rop_nr = (nv_rd32(dev, 0x409604) & 0x001f0000) >> 16;
 832         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 833                 priv->tp_nr[gpc] = nv_rd32(dev, GPC_UNIT(gpc, 0x2608));
 834                 priv->tp_total += priv->tp_nr[gpc];
 835         }
 836
 837         /*XXX: these need figuring out... */
 838         switch (dev_priv->chipset) {
 839         case 0xc0:
 840                 if (priv->tp_total == 11) { /* 465, 3/4/4/0, 4 */
 841                         priv->magic_not_rop_nr = 0x07;
 842                 } else
 843                 if (priv->tp_total == 14) { /* 470, 3/3/4/4, 5 */
 844                         priv->magic_not_rop_nr = 0x05;
 845                 } else
 846                 if (priv->tp_total == 15) { /* 480, 3/4/4/4, 6 */
 847                         priv->magic_not_rop_nr = 0x06;
 848                 }
 849                 break;
 850         case 0xc3: /* 450, 4/0/0/0, 2 */
 851                 priv->magic_not_rop_nr = 0x03;
 852                 break;
 853         case 0xc4: /* 460, 3/4/0/0, 4 */
 854                 priv->magic_not_rop_nr = 0x01;
 855                 break;
 856         case 0xc1: /* 2/0/0/0, 1 */
 857                 priv->magic_not_rop_nr = 0x01;
 858                 break;
 859         case 0xc8: /* 4/4/3/4, 5 */
 860                 priv->magic_not_rop_nr = 0x06;
 861                 break;
 862         case 0xce: /* 4/4/0/0, 4 */
 863                 priv->magic_not_rop_nr = 0x03;
 864                 break;
 865         }
 866
 867         if (!priv->magic_not_rop_nr) {
 868                 NV_ERROR(dev, "PGRAPH: unknown config: %d/%d/%d/%d, %d\n",
 869                          priv->tp_nr[0], priv->tp_nr[1], priv->tp_nr[2],
 870                          priv->tp_nr[3], priv->rop_nr);
 871                 /* use 0xc3's values... */
 872                 priv->magic_not_rop_nr = 0x03;
 873         }
 874
 875         NVOBJ_CLASS(dev, 0x902d, GR); /* 2D */
 876         NVOBJ_CLASS(dev, 0x9039, GR); /* M2MF */
 877         NVOBJ_MTHD (dev, 0x9039, 0x0500, nvc0_graph_mthd_page_flip);
 878         NVOBJ_CLASS(dev, 0x9097, GR); /* 3D */
 879         if (fermi >= 0x9197)
 880                 NVOBJ_CLASS(dev, 0x9197, GR); /* 3D (NVC1-) */
 881         if (fermi >= 0x9297)
 882                 NVOBJ_CLASS(dev, 0x9297, GR); /* 3D (NVC8-) */
 883         NVOBJ_CLASS(dev, 0x90c0, GR); /* COMPUTE */
 884         return 0;
 885
 886 error:
 887         nvc0_graph_destroy(dev, NVOBJ_ENGINE_GR);
 888         return ret;
 889 }
 890
 891 MODULE_FIRMWARE("nouveau/nvc0_fuc409c");
 892 MODULE_FIRMWARE("nouveau/nvc0_fuc409d");
 893 MODULE_FIRMWARE("nouveau/nvc0_fuc41ac");
 894 MODULE_FIRMWARE("nouveau/nvc0_fuc41ad");
 895 MODULE_FIRMWARE("nouveau/nvc3_fuc409c");
 896 MODULE_FIRMWARE("nouveau/nvc3_fuc409d");
 897 MODULE_FIRMWARE("nouveau/nvc3_fuc41ac");
 898 MODULE_FIRMWARE("nouveau/nvc3_fuc41ad");
 899 MODULE_FIRMWARE("nouveau/nvc4_fuc409c");
 900 MODULE_FIRMWARE("nouveau/nvc4_fuc409d");
 901 MODULE_FIRMWARE("nouveau/nvc4_fuc41ac");
 902 MODULE_FIRMWARE("nouveau/nvc4_fuc41ad");
 903 MODULE_FIRMWARE("nouveau/fuc409c");
 904 MODULE_FIRMWARE("nouveau/fuc409d");
 905 MODULE_FIRMWARE("nouveau/fuc41ac");
 906 MODULE_FIRMWARE("nouveau/fuc41ad");