drivers/gpu/drm/nouveau/nouveau_irq.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drm.h"
  36 #include "nouveau_drv.h"
  37 #include "nouveau_reg.h"
  38 #include "nouveau_ramht.h"
  39 #include <linux/ratelimit.h>
  40
  41 /* needed for hotplug irq */
  42 #include "nouveau_connector.h"
  43 #include "nv50_display.h"
  44
  45 static DEFINE_RATELIMIT_STATE(nouveau_ratelimit_state, 3 * HZ, 20);
  46
  47 static int nouveau_ratelimit(void)
  48 {
  49         return __ratelimit(&nouveau_ratelimit_state);
  50 }
  51
  52 void
  53 nouveau_irq_preinstall(struct drm_device *dev)
  54 {
  55         struct drm_nouveau_private *dev_priv = dev->dev_private;
  56
  57         /* Master disable */
  58         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  59
  60         if (dev_priv->card_type >= NV_50) {
  61                 INIT_WORK(&dev_priv->irq_work, nv50_display_irq_handler_bh);
  62                 INIT_WORK(&dev_priv->hpd_work, nv50_display_irq_hotplug_bh);
  63                 spin_lock_init(&dev_priv->hpd_state.lock);
  64                 INIT_LIST_HEAD(&dev_priv->vbl_waiting);
  65         }
  66 }
  67
  68 int
  69 nouveau_irq_postinstall(struct drm_device *dev)
  70 {
  71         /* Master enable */
  72         nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE);
  73         return 0;
  74 }
  75
  76 void
  77 nouveau_irq_uninstall(struct drm_device *dev)
  78 {
  79         /* Master disable */
  80         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  81 }
  82
  83 static int
  84 nouveau_call_method(struct nouveau_channel *chan, int class, int mthd, int data)
  85 {
  86         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
  87         struct nouveau_pgraph_object_method *grm;
  88         struct nouveau_pgraph_object_class *grc;
  89
  90         grc = dev_priv->engine.graph.grclass;
  91         while (grc->id) {
  92                 if (grc->id == class)
  93                         break;
  94                 grc++;
  95         }
  96
  97         if (grc->id != class || !grc->methods)
  98                 return -ENOENT;
  99
 100         grm = grc->methods;
 101         while (grm->id) {
 102                 if (grm->id == mthd)
 103                         return grm->exec(chan, class, mthd, data);
 104                 grm++;
 105         }
 106
 107         return -ENOENT;
 108 }
 109
 110 static bool
 111 nouveau_fifo_swmthd(struct nouveau_channel *chan, uint32_t addr, uint32_t data)
 112 {
 113         struct drm_device *dev = chan->dev;
 114         const int subc = (addr >> 13) & 0x7;
 115         const int mthd = addr & 0x1ffc;
 116
 117         if (mthd == 0x0000) {
 118                 struct nouveau_gpuobj *gpuobj;
 119
 120                 gpuobj = nouveau_ramht_find(chan, data);
 121                 if (!gpuobj)
 122                         return false;
 123
 124                 if (gpuobj->engine != NVOBJ_ENGINE_SW)
 125                         return false;
 126
 127                 chan->sw_subchannel[subc] = gpuobj->class;
 128                 nv_wr32(dev, NV04_PFIFO_CACHE1_ENGINE, nv_rd32(dev,
 129                         NV04_PFIFO_CACHE1_ENGINE) & ~(0xf << subc * 4));
 130                 return true;
 131         }
 132
 133         /* hw object */
 134         if (nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE) & (1 << (subc*4)))
 135                 return false;
 136
 137         if (nouveau_call_method(chan, chan->sw_subchannel[subc], mthd, data))
 138                 return false;
 139
 140         return true;
 141 }
 142
 143 static void
 144 nouveau_fifo_irq_handler(struct drm_device *dev)
 145 {
 146         struct drm_nouveau_private *dev_priv = dev->dev_private;
 147         struct nouveau_engine *engine = &dev_priv->engine;
 148         uint32_t status, reassign;
 149         int cnt = 0;
 150
 151         reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
 152         while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
 153                 struct nouveau_channel *chan = NULL;
 154                 uint32_t chid, get;
 155
 156                 nv_wr32(dev, NV03_PFIFO_CACHES, 0);
 157
 158                 chid = engine->fifo.channel_id(dev);
 159                 if (chid >= 0 && chid < engine->fifo.channels)
 160                         chan = dev_priv->fifos[chid];
 161                 get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
 162
 163                 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
 164                         uint32_t mthd, data;
 165                         int ptr;
 166
 167                         /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
 168                          * wrapping on my G80 chips, but CACHE1 isn't big
 169                          * enough for this much data.. Tests show that it
 170                          * wraps around to the start at GET=0x800.. No clue
 171                          * as to why..
 172                          */
 173                         ptr = (get & 0x7ff) >> 2;
 174
 175                         if (dev_priv->card_type < NV_40) {
 176                                 mthd = nv_rd32(dev,
 177                                         NV04_PFIFO_CACHE1_METHOD(ptr));
 178                                 data = nv_rd32(dev,
 179                                         NV04_PFIFO_CACHE1_DATA(ptr));
 180                         } else {
 181                                 mthd = nv_rd32(dev,
 182                                         NV40_PFIFO_CACHE1_METHOD(ptr));
 183                                 data = nv_rd32(dev,
 184                                         NV40_PFIFO_CACHE1_DATA(ptr));
 185                         }
 186
 187                         if (!chan || !nouveau_fifo_swmthd(chan, mthd, data)) {
 188                                 NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
 189                                              "Mthd 0x%04x Data 0x%08x\n",
 190                                         chid, (mthd >> 13) & 7, mthd & 0x1ffc,
 191                                         data);
 192                         }
 193
 194                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
 195                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 196                                                 NV_PFIFO_INTR_CACHE_ERROR);
 197
 198                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 199                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
 200                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 201                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 202                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
 203                         nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
 204
 205                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
 206                                 nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
 207                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 208
 209                         status &= ~NV_PFIFO_INTR_CACHE_ERROR;
 210                 }
 211
 212                 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
 213                         u32 dma_get = nv_rd32(dev, 0x003244);
 214                         u32 dma_put = nv_rd32(dev, 0x003240);
 215                         u32 push = nv_rd32(dev, 0x003220);
 216                         u32 state = nv_rd32(dev, 0x003228);
 217
 218                         if (dev_priv->card_type == NV_50) {
 219                                 u32 ho_get = nv_rd32(dev, 0x003328);
 220                                 u32 ho_put = nv_rd32(dev, 0x003320);
 221                                 u32 ib_get = nv_rd32(dev, 0x003334);
 222                                 u32 ib_put = nv_rd32(dev, 0x003330);
 223
 224                                 if (nouveau_ratelimit())
 225                                         NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
 226                                              "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
 227                                              "State 0x%08x Push 0x%08x\n",
 228                                                 chid, ho_get, dma_get, ho_put,
 229                                                 dma_put, ib_get, ib_put, state,
 230                                                 push);
 231
 232                                 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
 233                                 nv_wr32(dev, 0x003364, 0x00000000);
 234                                 if (dma_get != dma_put || ho_get != ho_put) {
 235                                         nv_wr32(dev, 0x003244, dma_put);
 236                                         nv_wr32(dev, 0x003328, ho_put);
 237                                 } else
 238                                 if (ib_get != ib_put) {
 239                                         nv_wr32(dev, 0x003334, ib_put);
 240                                 }
 241                         } else {
 242                                 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
 243                                              "Put 0x%08x State 0x%08x Push 0x%08x\n",
 244                                         chid, dma_get, dma_put, state, push);
 245
 246                                 if (dma_get != dma_put)
 247                                         nv_wr32(dev, 0x003244, dma_put);
 248                         }
 249
 250                         nv_wr32(dev, 0x003228, 0x00000000);
 251                         nv_wr32(dev, 0x003220, 0x00000001);
 252                         nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
 253                         status &= ~NV_PFIFO_INTR_DMA_PUSHER;
 254                 }
 255
 256                 if (status & NV_PFIFO_INTR_SEMAPHORE) {
 257                         uint32_t sem;
 258
 259                         status &= ~NV_PFIFO_INTR_SEMAPHORE;
 260                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 261                                 NV_PFIFO_INTR_SEMAPHORE);
 262
 263                         sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
 264                         nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
 265
 266                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 267                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 268                 }
 269
 270                 if (dev_priv->card_type == NV_50) {
 271                         if (status & 0x00000010) {
 272                                 nv50_fb_vm_trap(dev, 1, "PFIFO_BAR_FAULT");
 273                                 status &= ~0x00000010;
 274                                 nv_wr32(dev, 0x002100, 0x00000010);
 275                         }
 276                 }
 277
 278                 if (status) {
 279                         if (nouveau_ratelimit())
 280                                 NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
 281                                         status, chid);
 282                         nv_wr32(dev, NV03_PFIFO_INTR_0, status);
 283                         status = 0;
 284                 }
 285
 286                 nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
 287         }
 288
 289         if (status) {
 290                 NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
 291                 nv_wr32(dev, 0x2140, 0);
 292                 nv_wr32(dev, 0x140, 0);
 293         }
 294
 295         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
 296 }
 297
 298 struct nouveau_bitfield_names {
 299         uint32_t mask;
 300         const char *name;
 301 };
 302
 303 static struct nouveau_bitfield_names nstatus_names[] =
 304 {
 305         { NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 306         { NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 307         { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 308         { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
 309 };
 310
 311 static struct nouveau_bitfield_names nstatus_names_nv10[] =
 312 {
 313         { NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 314         { NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 315         { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 316         { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
 317 };
 318
 319 static struct nouveau_bitfield_names nsource_names[] =
 320 {
 321         { NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
 322         { NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
 323         { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
 324         { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
 325         { NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
 326         { NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
 327         { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
 328         { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
 329         { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
 330         { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
 331         { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
 332         { NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
 333         { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
 334         { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
 335         { NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
 336         { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
 337         { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
 338         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
 339         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
 340 };
 341
 342 static void
 343 nouveau_print_bitfield_names_(uint32_t value,
 344                                 const struct nouveau_bitfield_names *namelist,
 345                                 const int namelist_len)
 346 {
 347         /*
 348          * Caller must have already printed the KERN_* log level for us.
 349          * Also the caller is responsible for adding the newline.
 350          */
 351         int i;
 352         for (i = 0; i < namelist_len; ++i) {
 353                 uint32_t mask = namelist[i].mask;
 354                 if (value & mask) {
 355                         printk(" %s", namelist[i].name);
 356                         value &= ~mask;
 357                 }
 358         }
 359         if (value)
 360                 printk(" (unknown bits 0x%08x)", value);
 361 }
 362 #define nouveau_print_bitfield_names(val, namelist) \
 363         nouveau_print_bitfield_names_((val), (namelist), ARRAY_SIZE(namelist))
 364
 365 struct nouveau_enum_names {
 366         uint32_t value;
 367         const char *name;
 368 };
 369
 370 static void
 371 nouveau_print_enum_names_(uint32_t value,
 372                                 const struct nouveau_enum_names *namelist,
 373                                 const int namelist_len)
 374 {
 375         /*
 376          * Caller must have already printed the KERN_* log level for us.
 377          * Also the caller is responsible for adding the newline.
 378          */
 379         int i;
 380         for (i = 0; i < namelist_len; ++i) {
 381                 if (value == namelist[i].value) {
 382                         printk("%s", namelist[i].name);
 383                         return;
 384                 }
 385         }
 386         printk("unknown value 0x%08x", value);
 387 }
 388 #define nouveau_print_enum_names(val, namelist) \
 389         nouveau_print_enum_names_((val), (namelist), ARRAY_SIZE(namelist))
 390
 391 static int
 392 nouveau_graph_chid_from_grctx(struct drm_device *dev)
 393 {
 394         struct drm_nouveau_private *dev_priv = dev->dev_private;
 395         uint32_t inst;
 396         int i;
 397
 398         if (dev_priv->card_type < NV_40)
 399                 return dev_priv->engine.fifo.channels;
 400         else
 401         if (dev_priv->card_type < NV_50) {
 402                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 4;
 403
 404                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 405                         struct nouveau_channel *chan = dev_priv->fifos[i];
 406
 407                         if (!chan || !chan->ramin_grctx)
 408                                 continue;
 409
 410                         if (inst == chan->ramin_grctx->pinst)
 411                                 break;
 412                 }
 413         } else {
 414                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 12;
 415
 416                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 417                         struct nouveau_channel *chan = dev_priv->fifos[i];
 418
 419                         if (!chan || !chan->ramin)
 420                                 continue;
 421
 422                         if (inst == chan->ramin->vinst)
 423                                 break;
 424                 }
 425         }
 426
 427
 428         return i;
 429 }
 430
 431 static int
 432 nouveau_graph_trapped_channel(struct drm_device *dev, int *channel_ret)
 433 {
 434         struct drm_nouveau_private *dev_priv = dev->dev_private;
 435         struct nouveau_engine *engine = &dev_priv->engine;
 436         int channel;
 437
 438         if (dev_priv->card_type < NV_10)
 439                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 24) & 0xf;
 440         else
 441         if (dev_priv->card_type < NV_40)
 442                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
 443         else
 444                 channel = nouveau_graph_chid_from_grctx(dev);
 445
 446         if (channel >= engine->fifo.channels || !dev_priv->fifos[channel]) {
 447                 NV_ERROR(dev, "AIII, invalid/inactive channel id %d\n", channel);
 448                 return -EINVAL;
 449         }
 450
 451         *channel_ret = channel;
 452         return 0;
 453 }
 454
 455 struct nouveau_pgraph_trap {
 456         int channel;
 457         int class;
 458         int subc, mthd, size;
 459         uint32_t data, data2;
 460         uint32_t nsource, nstatus;
 461 };
 462
 463 static void
 464 nouveau_graph_trap_info(struct drm_device *dev,
 465                         struct nouveau_pgraph_trap *trap)
 466 {
 467         struct drm_nouveau_private *dev_priv = dev->dev_private;
 468         uint32_t address;
 469
 470         trap->nsource = trap->nstatus = 0;
 471         if (dev_priv->card_type < NV_50) {
 472                 trap->nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 473                 trap->nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
 474         }
 475
 476         if (nouveau_graph_trapped_channel(dev, &trap->channel))
 477                 trap->channel = -1;
 478         address = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
 479
 480         trap->mthd = address & 0x1FFC;
 481         trap->data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
 482         if (dev_priv->card_type < NV_10) {
 483                 trap->subc  = (address >> 13) & 0x7;
 484         } else {
 485                 trap->subc  = (address >> 16) & 0x7;
 486                 trap->data2 = nv_rd32(dev, NV10_PGRAPH_TRAPPED_DATA_HIGH);
 487         }
 488
 489         if (dev_priv->card_type < NV_10)
 490                 trap->class = nv_rd32(dev, 0x400180 + trap->subc*4) & 0xFF;
 491         else if (dev_priv->card_type < NV_40)
 492                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFF;
 493         else if (dev_priv->card_type < NV_50)
 494                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFFF;
 495         else
 496                 trap->class = nv_rd32(dev, 0x400814);
 497 }
 498
 499 static void
 500 nouveau_graph_dump_trap_info(struct drm_device *dev, const char *id,
 501                              struct nouveau_pgraph_trap *trap)
 502 {
 503         struct drm_nouveau_private *dev_priv = dev->dev_private;
 504         uint32_t nsource = trap->nsource, nstatus = trap->nstatus;
 505
 506         if (dev_priv->card_type < NV_50) {
 507                 NV_INFO(dev, "%s - nSource:", id);
 508                 nouveau_print_bitfield_names(nsource, nsource_names);
 509                 printk(", nStatus:");
 510                 if (dev_priv->card_type < NV_10)
 511                         nouveau_print_bitfield_names(nstatus, nstatus_names);
 512                 else
 513                         nouveau_print_bitfield_names(nstatus, nstatus_names_nv10);
 514                 printk("\n");
 515         }
 516
 517         NV_INFO(dev, "%s - Ch %d/%d Class 0x%04x Mthd 0x%04x "
 518                                         "Data 0x%08x:0x%08x\n",
 519                                         id, trap->channel, trap->subc,
 520                                         trap->class, trap->mthd,
 521                                         trap->data2, trap->data);
 522 }
 523
 524 static int
 525 nouveau_pgraph_intr_swmthd(struct drm_device *dev,
 526                            struct nouveau_pgraph_trap *trap)
 527 {
 528         struct drm_nouveau_private *dev_priv = dev->dev_private;
 529
 530         if (trap->channel < 0 ||
 531             trap->channel >= dev_priv->engine.fifo.channels ||
 532             !dev_priv->fifos[trap->channel])
 533                 return -ENODEV;
 534
 535         return nouveau_call_method(dev_priv->fifos[trap->channel],
 536                                    trap->class, trap->mthd, trap->data);
 537 }
 538
 539 static inline void
 540 nouveau_pgraph_intr_notify(struct drm_device *dev, uint32_t nsource)
 541 {
 542         struct nouveau_pgraph_trap trap;
 543         int unhandled = 0;
 544
 545         nouveau_graph_trap_info(dev, &trap);
 546
 547         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 548                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 549                         unhandled = 1;
 550         } else {
 551                 unhandled = 1;
 552         }
 553
 554         if (unhandled)
 555                 nouveau_graph_dump_trap_info(dev, "PGRAPH_NOTIFY", &trap);
 556 }
 557
 558
 559 static inline void
 560 nouveau_pgraph_intr_error(struct drm_device *dev, uint32_t nsource)
 561 {
 562         struct nouveau_pgraph_trap trap;
 563         int unhandled = 0;
 564
 565         nouveau_graph_trap_info(dev, &trap);
 566         trap.nsource = nsource;
 567
 568         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 569                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 570                         unhandled = 1;
 571         } else if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
 572                 uint32_t v = nv_rd32(dev, 0x402000);
 573                 nv_wr32(dev, 0x402000, v);
 574
 575                 /* dump the error anyway for now: it's useful for
 576                    Gallium development */
 577                 unhandled = 1;
 578         } else {
 579                 unhandled = 1;
 580         }
 581
 582         if (unhandled && nouveau_ratelimit())
 583                 nouveau_graph_dump_trap_info(dev, "PGRAPH_ERROR", &trap);
 584 }
 585
 586 static inline void
 587 nouveau_pgraph_intr_context_switch(struct drm_device *dev)
 588 {
 589         struct drm_nouveau_private *dev_priv = dev->dev_private;
 590         struct nouveau_engine *engine = &dev_priv->engine;
 591         uint32_t chid;
 592
 593         chid = engine->fifo.channel_id(dev);
 594         NV_DEBUG(dev, "PGRAPH context switch interrupt channel %x\n", chid);
 595
 596         switch (dev_priv->card_type) {
 597         case NV_04:
 598                 nv04_graph_context_switch(dev);
 599                 break;
 600         case NV_10:
 601                 nv10_graph_context_switch(dev);
 602                 break;
 603         default:
 604                 NV_ERROR(dev, "Context switch not implemented\n");
 605                 break;
 606         }
 607 }
 608
 609 static void
 610 nouveau_pgraph_irq_handler(struct drm_device *dev)
 611 {
 612         uint32_t status;
 613
 614         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
 615                 uint32_t nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 616
 617                 if (status & NV_PGRAPH_INTR_NOTIFY) {
 618                         nouveau_pgraph_intr_notify(dev, nsource);
 619
 620                         status &= ~NV_PGRAPH_INTR_NOTIFY;
 621                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_NOTIFY);
 622                 }
 623
 624                 if (status & NV_PGRAPH_INTR_ERROR) {
 625                         nouveau_pgraph_intr_error(dev, nsource);
 626
 627                         status &= ~NV_PGRAPH_INTR_ERROR;
 628                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_ERROR);
 629                 }
 630
 631                 if (status & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
 632                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
 633                         nv_wr32(dev, NV03_PGRAPH_INTR,
 634                                  NV_PGRAPH_INTR_CONTEXT_SWITCH);
 635
 636                         nouveau_pgraph_intr_context_switch(dev);
 637                 }
 638
 639                 if (status) {
 640                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n", status);
 641                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
 642                 }
 643
 644                 if ((nv_rd32(dev, NV04_PGRAPH_FIFO) & (1 << 0)) == 0)
 645                         nv_wr32(dev, NV04_PGRAPH_FIFO, 1);
 646         }
 647
 648         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
 649 }
 650
 651 static struct nouveau_enum_names nv50_mp_exec_error_names[] =
 652 {
 653         { 3, "STACK_UNDERFLOW" },
 654         { 4, "QUADON_ACTIVE" },
 655         { 8, "TIMEOUT" },
 656         { 0x10, "INVALID_OPCODE" },
 657         { 0x40, "BREAKPOINT" },
 658 };
 659
 660 static void
 661 nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
 662 {
 663         struct drm_nouveau_private *dev_priv = dev->dev_private;
 664         uint32_t units = nv_rd32(dev, 0x1540);
 665         uint32_t addr, mp10, status, pc, oplow, ophigh;
 666         int i;
 667         int mps = 0;
 668         for (i = 0; i < 4; i++) {
 669                 if (!(units & 1 << (i+24)))
 670                         continue;
 671                 if (dev_priv->chipset < 0xa0)
 672                         addr = 0x408200 + (tpid << 12) + (i << 7);
 673                 else
 674                         addr = 0x408100 + (tpid << 11) + (i << 7);
 675                 mp10 = nv_rd32(dev, addr + 0x10);
 676                 status = nv_rd32(dev, addr + 0x14);
 677                 if (!status)
 678                         continue;
 679                 if (display) {
 680                         nv_rd32(dev, addr + 0x20);
 681                         pc = nv_rd32(dev, addr + 0x24);
 682                         oplow = nv_rd32(dev, addr + 0x70);
 683                         ophigh= nv_rd32(dev, addr + 0x74);
 684                         NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
 685                                         "TP %d MP %d: ", tpid, i);
 686                         nouveau_print_enum_names(status,
 687                                         nv50_mp_exec_error_names);
 688                         printk(" at %06x warp %d, opcode %08x %08x\n",
 689                                         pc&0xffffff, pc >> 24,
 690                                         oplow, ophigh);
 691                 }
 692                 nv_wr32(dev, addr + 0x10, mp10);
 693                 nv_wr32(dev, addr + 0x14, 0);
 694                 mps++;
 695         }
 696         if (!mps && display)
 697                 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
 698                                 "No MPs claiming errors?\n", tpid);
 699 }
 700
 701 static void
 702 nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
 703                 uint32_t ustatus_new, int display, const char *name)
 704 {
 705         struct drm_nouveau_private *dev_priv = dev->dev_private;
 706         int tps = 0;
 707         uint32_t units = nv_rd32(dev, 0x1540);
 708         int i, r;
 709         uint32_t ustatus_addr, ustatus;
 710         for (i = 0; i < 16; i++) {
 711                 if (!(units & (1 << i)))
 712                         continue;
 713                 if (dev_priv->chipset < 0xa0)
 714                         ustatus_addr = ustatus_old + (i << 12);
 715                 else
 716                         ustatus_addr = ustatus_new + (i << 11);
 717                 ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
 718                 if (!ustatus)
 719                         continue;
 720                 tps++;
 721                 switch (type) {
 722                 case 6: /* texture error... unknown for now */
 723                         nv50_fb_vm_trap(dev, display, name);
 724                         if (display) {
 725                                 NV_ERROR(dev, "magic set %d:\n", i);
 726                                 for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
 727                                         NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
 728                                                 nv_rd32(dev, r));
 729                         }
 730                         break;
 731                 case 7: /* MP error */
 732                         if (ustatus & 0x00010000) {
 733                                 nv50_pgraph_mp_trap(dev, i, display);
 734                                 ustatus &= ~0x00010000;
 735                         }
 736                         break;
 737                 case 8: /* TPDMA error */
 738                         {
 739                         uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
 740                         uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
 741                         uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
 742                         uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
 743                         uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
 744                         uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
 745                         uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
 746                         nv50_fb_vm_trap(dev, display, name);
 747                         /* 2d engine destination */
 748                         if (ustatus & 0x00000010) {
 749                                 if (display) {
 750                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
 751                                                         i, e14, e10);
 752                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 753                                                         i, e0c, e18, e1c, e20, e24);
 754                                 }
 755                                 ustatus &= ~0x00000010;
 756                         }
 757                         /* Render target */
 758                         if (ustatus & 0x00000040) {
 759                                 if (display) {
 760                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
 761                                                         i, e14, e10);
 762                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 763                                                         i, e0c, e18, e1c, e20, e24);
 764                                 }
 765                                 ustatus &= ~0x00000040;
 766                         }
 767                         /* CUDA memory: l[], g[] or stack. */
 768                         if (ustatus & 0x00000080) {
 769                                 if (display) {
 770                                         if (e18 & 0x80000000) {
 771                                                 /* g[] read fault? */
 772                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
 773                                                                 i, e14, e10 | ((e18 >> 24) & 0x1f));
 774                                                 e18 &= ~0x1f000000;
 775                                         } else if (e18 & 0xc) {
 776                                                 /* g[] write fault? */
 777                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
 778                                                                 i, e14, e10 | ((e18 >> 7) & 0x1f));
 779                                                 e18 &= ~0x00000f80;
 780                                         } else {
 781                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
 782                                                                 i, e14, e10);
 783                                         }
 784                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 785                                                         i, e0c, e18, e1c, e20, e24);
 786                                 }
 787                                 ustatus &= ~0x00000080;
 788                         }
 789                         }
 790                         break;
 791                 }
 792                 if (ustatus) {
 793                         if (display)
 794                                 NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
 795                 }
 796                 nv_wr32(dev, ustatus_addr, 0xc0000000);
 797         }
 798
 799         if (!tps && display)
 800                 NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
 801 }
 802
 803 static void
 804 nv50_pgraph_trap_handler(struct drm_device *dev)
 805 {
 806         struct nouveau_pgraph_trap trap;
 807         uint32_t status = nv_rd32(dev, 0x400108);
 808         uint32_t ustatus;
 809         int display = nouveau_ratelimit();
 810
 811
 812         if (!status && display) {
 813                 nouveau_graph_trap_info(dev, &trap);
 814                 nouveau_graph_dump_trap_info(dev, "PGRAPH_TRAP", &trap);
 815                 NV_INFO(dev, "PGRAPH_TRAP - no units reporting traps?\n");
 816         }
 817
 818         /* DISPATCH: Relays commands to other units and handles NOTIFY,
 819          * COND, QUERY. If you get a trap from it, the command is still stuck
 820          * in DISPATCH and you need to do something about it. */
 821         if (status & 0x001) {
 822                 ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
 823                 if (!ustatus && display) {
 824                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
 825                 }
 826
 827                 /* Known to be triggered by screwed up NOTIFY and COND... */
 828                 if (ustatus & 0x00000001) {
 829                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_FAULT");
 830                         nv_wr32(dev, 0x400500, 0);
 831                         if (nv_rd32(dev, 0x400808) & 0x80000000) {
 832                                 if (display) {
 833                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 834                                                 trap.channel = -1;
 835                                         trap.class = nv_rd32(dev, 0x400814);
 836                                         trap.mthd = nv_rd32(dev, 0x400808) & 0x1ffc;
 837                                         trap.subc = (nv_rd32(dev, 0x400808) >> 16) & 0x7;
 838                                         trap.data = nv_rd32(dev, 0x40080c);
 839                                         trap.data2 = nv_rd32(dev, 0x400810);
 840                                         nouveau_graph_dump_trap_info(dev,
 841                                                         "PGRAPH_TRAP_DISPATCH_FAULT", &trap);
 842                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400808: %08x\n", nv_rd32(dev, 0x400808));
 843                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400848: %08x\n", nv_rd32(dev, 0x400848));
 844                                 }
 845                                 nv_wr32(dev, 0x400808, 0);
 846                         } else if (display) {
 847                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - No stuck command?\n");
 848                         }
 849                         nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
 850                         nv_wr32(dev, 0x400848, 0);
 851                         ustatus &= ~0x00000001;
 852                 }
 853                 if (ustatus & 0x00000002) {
 854                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_QUERY");
 855                         nv_wr32(dev, 0x400500, 0);
 856                         if (nv_rd32(dev, 0x40084c) & 0x80000000) {
 857                                 if (display) {
 858                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 859                                                 trap.channel = -1;
 860                                         trap.class = nv_rd32(dev, 0x400814);
 861                                         trap.mthd = nv_rd32(dev, 0x40084c) & 0x1ffc;
 862                                         trap.subc = (nv_rd32(dev, 0x40084c) >> 16) & 0x7;
 863                                         trap.data = nv_rd32(dev, 0x40085c);
 864                                         trap.data2 = 0;
 865                                         nouveau_graph_dump_trap_info(dev,
 866                                                         "PGRAPH_TRAP_DISPATCH_QUERY", &trap);
 867                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - 40084c: %08x\n", nv_rd32(dev, 0x40084c));
 868                                 }
 869                                 nv_wr32(dev, 0x40084c, 0);
 870                         } else if (display) {
 871                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - No stuck command?\n");
 872                         }
 873                         ustatus &= ~0x00000002;
 874                 }
 875                 if (ustatus && display)
 876                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - Unhandled ustatus 0x%08x\n", ustatus);
 877                 nv_wr32(dev, 0x400804, 0xc0000000);
 878                 nv_wr32(dev, 0x400108, 0x001);
 879                 status &= ~0x001;
 880         }
 881
 882         /* TRAPs other than dispatch use the "normal" trap regs. */
 883         if (status && display) {
 884                 nouveau_graph_trap_info(dev, &trap);
 885                 nouveau_graph_dump_trap_info(dev,
 886                                 "PGRAPH_TRAP", &trap);
 887         }
 888
 889         /* M2MF: Memory to memory copy engine. */
 890         if (status & 0x002) {
 891                 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
 892                 if (!ustatus && display) {
 893                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - no ustatus?\n");
 894                 }
 895                 if (ustatus & 0x00000001) {
 896                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_NOTIFY");
 897                         ustatus &= ~0x00000001;
 898                 }
 899                 if (ustatus & 0x00000002) {
 900                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_IN");
 901                         ustatus &= ~0x00000002;
 902                 }
 903                 if (ustatus & 0x00000004) {
 904                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_OUT");
 905                         ustatus &= ~0x00000004;
 906                 }
 907                 NV_INFO (dev, "PGRAPH_TRAP_M2MF - %08x %08x %08x %08x\n",
 908                                 nv_rd32(dev, 0x406804),
 909                                 nv_rd32(dev, 0x406808),
 910                                 nv_rd32(dev, 0x40680c),
 911                                 nv_rd32(dev, 0x406810));
 912                 if (ustatus && display)
 913                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - Unhandled ustatus 0x%08x\n", ustatus);
 914                 /* No sane way found yet -- just reset the bugger. */
 915                 nv_wr32(dev, 0x400040, 2);
 916                 nv_wr32(dev, 0x400040, 0);
 917                 nv_wr32(dev, 0x406800, 0xc0000000);
 918                 nv_wr32(dev, 0x400108, 0x002);
 919                 status &= ~0x002;
 920         }
 921
 922         /* VFETCH: Fetches data from vertex buffers. */
 923         if (status & 0x004) {
 924                 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
 925                 if (!ustatus && display) {
 926                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - no ustatus?\n");
 927                 }
 928                 if (ustatus & 0x00000001) {
 929                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_VFETCH_FAULT");
 930                         NV_INFO (dev, "PGRAPH_TRAP_VFETCH_FAULT - %08x %08x %08x %08x\n",
 931                                         nv_rd32(dev, 0x400c00),
 932                                         nv_rd32(dev, 0x400c08),
 933                                         nv_rd32(dev, 0x400c0c),
 934                                         nv_rd32(dev, 0x400c10));
 935                         ustatus &= ~0x00000001;
 936                 }
 937                 if (ustatus && display)
 938                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - Unhandled ustatus 0x%08x\n", ustatus);
 939                 nv_wr32(dev, 0x400c04, 0xc0000000);
 940                 nv_wr32(dev, 0x400108, 0x004);
 941                 status &= ~0x004;
 942         }
 943
 944         /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
 945         if (status & 0x008) {
 946                 ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
 947                 if (!ustatus && display) {
 948                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - no ustatus?\n");
 949                 }
 950                 if (ustatus & 0x00000001) {
 951                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_STRMOUT_FAULT");
 952                         NV_INFO (dev, "PGRAPH_TRAP_STRMOUT_FAULT - %08x %08x %08x %08x\n",
 953                                         nv_rd32(dev, 0x401804),
 954                                         nv_rd32(dev, 0x401808),
 955                                         nv_rd32(dev, 0x40180c),
 956                                         nv_rd32(dev, 0x401810));
 957                         ustatus &= ~0x00000001;
 958                 }
 959                 if (ustatus && display)
 960                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - Unhandled ustatus 0x%08x\n", ustatus);
 961                 /* No sane way found yet -- just reset the bugger. */
 962                 nv_wr32(dev, 0x400040, 0x80);
 963                 nv_wr32(dev, 0x400040, 0);
 964                 nv_wr32(dev, 0x401800, 0xc0000000);
 965                 nv_wr32(dev, 0x400108, 0x008);
 966                 status &= ~0x008;
 967         }
 968
 969         /* CCACHE: Handles code and c[] caches and fills them. */
 970         if (status & 0x010) {
 971                 ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
 972                 if (!ustatus && display) {
 973                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - no ustatus?\n");
 974                 }
 975                 if (ustatus & 0x00000001) {
 976                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_CCACHE_FAULT");
 977                         NV_INFO (dev, "PGRAPH_TRAP_CCACHE_FAULT - %08x %08x %08x %08x %08x %08x %08x\n",
 978                                         nv_rd32(dev, 0x405800),
 979                                         nv_rd32(dev, 0x405804),
 980                                         nv_rd32(dev, 0x405808),
 981                                         nv_rd32(dev, 0x40580c),
 982                                         nv_rd32(dev, 0x405810),
 983                                         nv_rd32(dev, 0x405814),
 984                                         nv_rd32(dev, 0x40581c));
 985                         ustatus &= ~0x00000001;
 986                 }
 987                 if (ustatus && display)
 988                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - Unhandled ustatus 0x%08x\n", ustatus);
 989                 nv_wr32(dev, 0x405018, 0xc0000000);
 990                 nv_wr32(dev, 0x400108, 0x010);
 991                 status &= ~0x010;
 992         }
 993
 994         /* Unknown, not seen yet... 0x402000 is the only trap status reg
 995          * remaining, so try to handle it anyway. Perhaps related to that
 996          * unknown DMA slot on tesla? */
 997         if (status & 0x20) {
 998                 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_UNKC04");
 999                 ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
1000                 if (display)
1001                         NV_INFO(dev, "PGRAPH_TRAP_UNKC04 - Unhandled ustatus 0x%08x\n", ustatus);
1002                 nv_wr32(dev, 0x402000, 0xc0000000);
1003                 /* no status modifiction on purpose */
1004         }
1005
1006         /* TEXTURE: CUDA texturing units */
1007         if (status & 0x040) {
1008                 nv50_pgraph_tp_trap (dev, 6, 0x408900, 0x408600, display,
1009                                 "PGRAPH_TRAP_TEXTURE");
1010                 nv_wr32(dev, 0x400108, 0x040);
1011                 status &= ~0x040;
1012         }
1013
1014         /* MP: CUDA execution engines. */
1015         if (status & 0x080) {
1016                 nv50_pgraph_tp_trap (dev, 7, 0x408314, 0x40831c, display,
1017                                 "PGRAPH_TRAP_MP");
1018                 nv_wr32(dev, 0x400108, 0x080);
1019                 status &= ~0x080;
1020         }
1021
1022         /* TPDMA:  Handles TP-initiated uncached memory accesses:
1023          * l[], g[], stack, 2d surfaces, render targets. */
1024         if (status & 0x100) {
1025                 nv50_pgraph_tp_trap (dev, 8, 0x408e08, 0x408708, display,
1026                                 "PGRAPH_TRAP_TPDMA");
1027                 nv_wr32(dev, 0x400108, 0x100);
1028                 status &= ~0x100;
1029         }
1030
1031         if (status) {
1032                 if (display)
1033                         NV_INFO(dev, "PGRAPH_TRAP - Unknown trap 0x%08x\n",
1034                                 status);
1035                 nv_wr32(dev, 0x400108, status);
1036         }
1037 }
1038
1039 /* There must be a *lot* of these. Will take some time to gather them up. */
1040 static struct nouveau_enum_names nv50_data_error_names[] =
1041 {
1042         { 4,    "INVALID_VALUE" },
1043         { 5,    "INVALID_ENUM" },
1044         { 8,    "INVALID_OBJECT" },
1045         { 0xc,  "INVALID_BITFIELD" },
1046         { 0x28, "MP_NO_REG_SPACE" },
1047         { 0x2b, "MP_BLOCK_SIZE_MISMATCH" },
1048 };
1049
1050 static void
1051 nv50_pgraph_irq_handler(struct drm_device *dev)
1052 {
1053         struct nouveau_pgraph_trap trap;
1054         int unhandled = 0;
1055         uint32_t status;
1056
1057         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
1058                 /* NOTIFY: You've set a NOTIFY an a command and it's done. */
1059                 if (status & 0x00000001) {
1060                         nouveau_graph_trap_info(dev, &trap);
1061                         if (nouveau_ratelimit())
1062                                 nouveau_graph_dump_trap_info(dev,
1063                                                 "PGRAPH_NOTIFY", &trap);
1064                         status &= ~0x00000001;
1065                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000001);
1066                 }
1067
1068                 /* COMPUTE_QUERY: Purpose and exact cause unknown, happens
1069                  * when you write 0x200 to 0x50c0 method 0x31c. */
1070                 if (status & 0x00000002) {
1071                         nouveau_graph_trap_info(dev, &trap);
1072                         if (nouveau_ratelimit())
1073                                 nouveau_graph_dump_trap_info(dev,
1074                                                 "PGRAPH_COMPUTE_QUERY", &trap);
1075                         status &= ~0x00000002;
1076                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000002);
1077                 }
1078
1079                 /* Unknown, never seen: 0x4 */
1080
1081                 /* ILLEGAL_MTHD: You used a wrong method for this class. */
1082                 if (status & 0x00000010) {
1083                         nouveau_graph_trap_info(dev, &trap);
1084                         if (nouveau_pgraph_intr_swmthd(dev, &trap))
1085                                 unhandled = 1;
1086                         if (unhandled && nouveau_ratelimit())
1087                                 nouveau_graph_dump_trap_info(dev,
1088                                                 "PGRAPH_ILLEGAL_MTHD", &trap);
1089                         status &= ~0x00000010;
1090                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000010);
1091                 }
1092
1093                 /* ILLEGAL_CLASS: You used a wrong class. */
1094                 if (status & 0x00000020) {
1095                         nouveau_graph_trap_info(dev, &trap);
1096                         if (nouveau_ratelimit())
1097                                 nouveau_graph_dump_trap_info(dev,
1098                                                 "PGRAPH_ILLEGAL_CLASS", &trap);
1099                         status &= ~0x00000020;
1100                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000020);
1101                 }
1102
1103                 /* DOUBLE_NOTIFY: You tried to set a NOTIFY on another NOTIFY. */
1104                 if (status & 0x00000040) {
1105                         nouveau_graph_trap_info(dev, &trap);
1106                         if (nouveau_ratelimit())
1107                                 nouveau_graph_dump_trap_info(dev,
1108                                                 "PGRAPH_DOUBLE_NOTIFY", &trap);
1109                         status &= ~0x00000040;
1110                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000040);
1111                 }
1112
1113                 /* CONTEXT_SWITCH: PGRAPH needs us to load a new context */
1114                 if (status & 0x00001000) {
1115                         nv_wr32(dev, 0x400500, 0x00000000);
1116                         nv_wr32(dev, NV03_PGRAPH_INTR,
1117                                 NV_PGRAPH_INTR_CONTEXT_SWITCH);
1118                         nv_wr32(dev, NV40_PGRAPH_INTR_EN, nv_rd32(dev,
1119                                 NV40_PGRAPH_INTR_EN) &
1120                                 ~NV_PGRAPH_INTR_CONTEXT_SWITCH);
1121                         nv_wr32(dev, 0x400500, 0x00010001);
1122
1123                         nv50_graph_context_switch(dev);
1124
1125                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1126                 }
1127
1128                 /* BUFFER_NOTIFY: Your m2mf transfer finished */
1129                 if (status & 0x00010000) {
1130                         nouveau_graph_trap_info(dev, &trap);
1131                         if (nouveau_ratelimit())
1132                                 nouveau_graph_dump_trap_info(dev,
1133                                                 "PGRAPH_BUFFER_NOTIFY", &trap);
1134                         status &= ~0x00010000;
1135                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00010000);
1136                 }
1137
1138                 /* DATA_ERROR: Invalid value for this method, or invalid
1139                  * state in current PGRAPH context for this operation */
1140                 if (status & 0x00100000) {
1141                         nouveau_graph_trap_info(dev, &trap);
1142                         if (nouveau_ratelimit()) {
1143                                 nouveau_graph_dump_trap_info(dev,
1144                                                 "PGRAPH_DATA_ERROR", &trap);
1145                                 NV_INFO (dev, "PGRAPH_DATA_ERROR - ");
1146                                 nouveau_print_enum_names(nv_rd32(dev, 0x400110),
1147                                                 nv50_data_error_names);
1148                                 printk("\n");
1149                         }
1150                         status &= ~0x00100000;
1151                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00100000);
1152                 }
1153
1154                 /* TRAP: Something bad happened in the middle of command
1155                  * execution.  Has a billion types, subtypes, and even
1156                  * subsubtypes. */
1157                 if (status & 0x00200000) {
1158                         nv50_pgraph_trap_handler(dev);
1159                         status &= ~0x00200000;
1160                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00200000);
1161                 }
1162
1163                 /* Unknown, never seen: 0x00400000 */
1164
1165                 /* SINGLE_STEP: Happens on every method if you turned on
1166                  * single stepping in 40008c */
1167                 if (status & 0x01000000) {
1168                         nouveau_graph_trap_info(dev, &trap);
1169                         if (nouveau_ratelimit())
1170                                 nouveau_graph_dump_trap_info(dev,
1171                                                 "PGRAPH_SINGLE_STEP", &trap);
1172                         status &= ~0x01000000;
1173                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x01000000);
1174                 }
1175
1176                 /* 0x02000000 happens when you pause a ctxprog...
1177                  * but the only way this can happen that I know is by
1178                  * poking the relevant MMIO register, and we don't
1179                  * do that. */
1180
1181                 if (status) {
1182                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n",
1183                                 status);
1184                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
1185                 }
1186
1187                 {
1188                         const int isb = (1 << 16) | (1 << 0);
1189
1190                         if ((nv_rd32(dev, 0x400500) & isb) != isb)
1191                                 nv_wr32(dev, 0x400500,
1192                                         nv_rd32(dev, 0x400500) | isb);
1193                 }
1194         }
1195
1196         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
1197         if (nv_rd32(dev, 0x400824) & (1 << 31))
1198                 nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
1199 }
1200
1201 static void
1202 nouveau_crtc_irq_handler(struct drm_device *dev, int crtc)
1203 {
1204         if (crtc & 1)
1205                 nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK);
1206
1207         if (crtc & 2)
1208                 nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK);
1209 }
1210
1211 irqreturn_t
1212 nouveau_irq_handler(DRM_IRQ_ARGS)
1213 {
1214         struct drm_device *dev = (struct drm_device *)arg;
1215         struct drm_nouveau_private *dev_priv = dev->dev_private;
1216         uint32_t status;
1217         unsigned long flags;
1218
1219         status = nv_rd32(dev, NV03_PMC_INTR_0);
1220         if (!status)
1221                 return IRQ_NONE;
1222
1223         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1224
1225         if (status & NV_PMC_INTR_0_PFIFO_PENDING) {
1226                 nouveau_fifo_irq_handler(dev);
1227                 status &= ~NV_PMC_INTR_0_PFIFO_PENDING;
1228         }
1229
1230         if (status & NV_PMC_INTR_0_PGRAPH_PENDING) {
1231                 if (dev_priv->card_type >= NV_50)
1232                         nv50_pgraph_irq_handler(dev);
1233                 else
1234                         nouveau_pgraph_irq_handler(dev);
1235
1236                 status &= ~NV_PMC_INTR_0_PGRAPH_PENDING;
1237         }
1238
1239         if (status & NV_PMC_INTR_0_CRTCn_PENDING) {
1240                 nouveau_crtc_irq_handler(dev, (status>>24)&3);
1241                 status &= ~NV_PMC_INTR_0_CRTCn_PENDING;
1242         }
1243
1244         if (status & (NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
1245                       NV_PMC_INTR_0_NV50_I2C_PENDING)) {
1246                 nv50_display_irq_handler(dev);
1247                 status &= ~(NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
1248                             NV_PMC_INTR_0_NV50_I2C_PENDING);
1249         }
1250
1251         if (status)
1252                 NV_ERROR(dev, "Unhandled PMC INTR status bits 0x%08x\n", status);
1253
1254         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1255
1256         return IRQ_HANDLED;
1257 }