drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #include <linux/sysrq.h>
  30 #include <linux/slab.h>
  31 #include "drmP.h"
  32 #include "drm.h"
  33 #include "i915_drm.h"
  34 #include "i915_drv.h"
  35 #include "i915_trace.h"
  36 #include "intel_drv.h"
  37
  38 #define MAX_NOPID ((u32)~0)
  39
  40 /**
  41  * Interrupts that are always left unmasked.
  42  *
  43  * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
  44  * we leave them always unmasked in IMR and then control enabling them through
  45  * PIPESTAT alone.
  46  */
  47 #define I915_INTERRUPT_ENABLE_FIX                       \
  48         (I915_ASLE_INTERRUPT |                          \
  49          I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |          \
  50          I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |          \
  51          I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |  \
  52          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |  \
  53          I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
  54
  55 /** Interrupts that we mask and unmask at runtime. */
  56 #define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT | I915_BSD_USER_INTERRUPT)
  57
  58 #define I915_PIPE_VBLANK_STATUS (PIPE_START_VBLANK_INTERRUPT_STATUS |\
  59                                  PIPE_VBLANK_INTERRUPT_STATUS)
  60
  61 #define I915_PIPE_VBLANK_ENABLE (PIPE_START_VBLANK_INTERRUPT_ENABLE |\
  62                                  PIPE_VBLANK_INTERRUPT_ENABLE)
  63
  64 #define DRM_I915_VBLANK_PIPE_ALL        (DRM_I915_VBLANK_PIPE_A | \
  65                                          DRM_I915_VBLANK_PIPE_B)
  66
  67 /* For display hotplug interrupt */
  68 static void
  69 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  70 {
  71         if ((dev_priv->irq_mask & mask) != 0) {
  72                 dev_priv->irq_mask &= ~mask;
  73                 I915_WRITE(DEIMR, dev_priv->irq_mask);
  74                 POSTING_READ(DEIMR);
  75         }
  76 }
  77
  78 static inline void
  79 ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  80 {
  81         if ((dev_priv->irq_mask & mask) != mask) {
  82                 dev_priv->irq_mask |= mask;
  83                 I915_WRITE(DEIMR, dev_priv->irq_mask);
  84                 POSTING_READ(DEIMR);
  85         }
  86 }
  87
  88 void
  89 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
  90 {
  91         if ((dev_priv->pipestat[pipe] & mask) != mask) {
  92                 u32 reg = PIPESTAT(pipe);
  93
  94                 dev_priv->pipestat[pipe] |= mask;
  95                 /* Enable the interrupt, clear any pending status */
  96                 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
  97                 POSTING_READ(reg);
  98         }
  99 }
 100
 101 void
 102 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 103 {
 104         if ((dev_priv->pipestat[pipe] & mask) != 0) {
 105                 u32 reg = PIPESTAT(pipe);
 106
 107                 dev_priv->pipestat[pipe] &= ~mask;
 108                 I915_WRITE(reg, dev_priv->pipestat[pipe]);
 109                 POSTING_READ(reg);
 110         }
 111 }
 112
 113 /**
 114  * intel_enable_asle - enable ASLE interrupt for OpRegion
 115  */
 116 void intel_enable_asle(struct drm_device *dev)
 117 {
 118         drm_i915_private_t *dev_priv = dev->dev_private;
 119         unsigned long irqflags;
 120
 121         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 122
 123         if (HAS_PCH_SPLIT(dev))
 124                 ironlake_enable_display_irq(dev_priv, DE_GSE);
 125         else {
 126                 i915_enable_pipestat(dev_priv, 1,
 127                                      PIPE_LEGACY_BLC_EVENT_ENABLE);
 128                 if (INTEL_INFO(dev)->gen >= 4)
 129                         i915_enable_pipestat(dev_priv, 0,
 130                                              PIPE_LEGACY_BLC_EVENT_ENABLE);
 131         }
 132
 133         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 134 }
 135
 136 /**
 137  * i915_pipe_enabled - check if a pipe is enabled
 138  * @dev: DRM device
 139  * @pipe: pipe to check
 140  *
 141  * Reading certain registers when the pipe is disabled can hang the chip.
 142  * Use this routine to make sure the PLL is running and the pipe is active
 143  * before reading such registers if unsure.
 144  */
 145 static int
 146 i915_pipe_enabled(struct drm_device *dev, int pipe)
 147 {
 148         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 149         return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
 150 }
 151
 152 /* Called from drm generic code, passed a 'crtc', which
 153  * we use as a pipe index
 154  */
 155 u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 156 {
 157         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 158         unsigned long high_frame;
 159         unsigned long low_frame;
 160         u32 high1, high2, low;
 161
 162         if (!i915_pipe_enabled(dev, pipe)) {
 163                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 164                                 "pipe %c\n", pipe_name(pipe));
 165                 return 0;
 166         }
 167
 168         high_frame = PIPEFRAME(pipe);
 169         low_frame = PIPEFRAMEPIXEL(pipe);
 170
 171         /*
 172          * High & low register fields aren't synchronized, so make sure
 173          * we get a low value that's stable across two reads of the high
 174          * register.
 175          */
 176         do {
 177                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 178                 low   = I915_READ(low_frame)  & PIPE_FRAME_LOW_MASK;
 179                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 180         } while (high1 != high2);
 181
 182         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 183         low >>= PIPE_FRAME_LOW_SHIFT;
 184         return (high1 << 8) | low;
 185 }
 186
 187 u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 188 {
 189         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 190         int reg = PIPE_FRMCOUNT_GM45(pipe);
 191
 192         if (!i915_pipe_enabled(dev, pipe)) {
 193                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 194                                  "pipe %c\n", pipe_name(pipe));
 195                 return 0;
 196         }
 197
 198         return I915_READ(reg);
 199 }
 200
 201 int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 202                              int *vpos, int *hpos)
 203 {
 204         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 205         u32 vbl = 0, position = 0;
 206         int vbl_start, vbl_end, htotal, vtotal;
 207         bool in_vbl = true;
 208         int ret = 0;
 209
 210         if (!i915_pipe_enabled(dev, pipe)) {
 211                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 212                                  "pipe %c\n", pipe_name(pipe));
 213                 return 0;
 214         }
 215
 216         /* Get vtotal. */
 217         vtotal = 1 + ((I915_READ(VTOTAL(pipe)) >> 16) & 0x1fff);
 218
 219         if (INTEL_INFO(dev)->gen >= 4) {
 220                 /* No obvious pixelcount register. Only query vertical
 221                  * scanout position from Display scan line register.
 222                  */
 223                 position = I915_READ(PIPEDSL(pipe));
 224
 225                 /* Decode into vertical scanout position. Don't have
 226                  * horizontal scanout position.
 227                  */
 228                 *vpos = position & 0x1fff;
 229                 *hpos = 0;
 230         } else {
 231                 /* Have access to pixelcount since start of frame.
 232                  * We can split this into vertical and horizontal
 233                  * scanout position.
 234                  */
 235                 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 236
 237                 htotal = 1 + ((I915_READ(HTOTAL(pipe)) >> 16) & 0x1fff);
 238                 *vpos = position / htotal;
 239                 *hpos = position - (*vpos * htotal);
 240         }
 241
 242         /* Query vblank area. */
 243         vbl = I915_READ(VBLANK(pipe));
 244
 245         /* Test position against vblank region. */
 246         vbl_start = vbl & 0x1fff;
 247         vbl_end = (vbl >> 16) & 0x1fff;
 248
 249         if ((*vpos < vbl_start) || (*vpos > vbl_end))
 250                 in_vbl = false;
 251
 252         /* Inside "upper part" of vblank area? Apply corrective offset: */
 253         if (in_vbl && (*vpos >= vbl_start))
 254                 *vpos = *vpos - vtotal;
 255
 256         /* Readouts valid? */
 257         if (vbl > 0)
 258                 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 259
 260         /* In vblank? */
 261         if (in_vbl)
 262                 ret |= DRM_SCANOUTPOS_INVBL;
 263
 264         return ret;
 265 }
 266
 267 int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 268                               int *max_error,
 269                               struct timeval *vblank_time,
 270                               unsigned flags)
 271 {
 272         struct drm_i915_private *dev_priv = dev->dev_private;
 273         struct drm_crtc *crtc;
 274
 275         if (pipe < 0 || pipe >= dev_priv->num_pipe) {
 276                 DRM_ERROR("Invalid crtc %d\n", pipe);
 277                 return -EINVAL;
 278         }
 279
 280         /* Get drm_crtc to timestamp: */
 281         crtc = intel_get_crtc_for_pipe(dev, pipe);
 282         if (crtc == NULL) {
 283                 DRM_ERROR("Invalid crtc %d\n", pipe);
 284                 return -EINVAL;
 285         }
 286
 287         if (!crtc->enabled) {
 288                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 289                 return -EBUSY;
 290         }
 291
 292         /* Helper routine in DRM core does all the work: */
 293         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 294                                                      vblank_time, flags,
 295                                                      crtc);
 296 }
 297
 298 /*
 299  * Handle hotplug events outside the interrupt handler proper.
 300  */
 301 static void i915_hotplug_work_func(struct work_struct *work)
 302 {
 303         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 304                                                     hotplug_work);
 305         struct drm_device *dev = dev_priv->dev;
 306         struct drm_mode_config *mode_config = &dev->mode_config;
 307         struct intel_encoder *encoder;
 308
 309         DRM_DEBUG_KMS("running encoder hotplug functions\n");
 310
 311         list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
 312                 if (encoder->hot_plug)
 313                         encoder->hot_plug(encoder);
 314
 315         /* Just fire off a uevent and let userspace tell us what to do */
 316         drm_helper_hpd_irq_event(dev);
 317 }
 318
 319 static void i915_handle_rps_change(struct drm_device *dev)
 320 {
 321         drm_i915_private_t *dev_priv = dev->dev_private;
 322         u32 busy_up, busy_down, max_avg, min_avg;
 323         u8 new_delay = dev_priv->cur_delay;
 324
 325         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 326         busy_up = I915_READ(RCPREVBSYTUPAVG);
 327         busy_down = I915_READ(RCPREVBSYTDNAVG);
 328         max_avg = I915_READ(RCBMAXAVG);
 329         min_avg = I915_READ(RCBMINAVG);
 330
 331         /* Handle RCS change request from hw */
 332         if (busy_up > max_avg) {
 333                 if (dev_priv->cur_delay != dev_priv->max_delay)
 334                         new_delay = dev_priv->cur_delay - 1;
 335                 if (new_delay < dev_priv->max_delay)
 336                         new_delay = dev_priv->max_delay;
 337         } else if (busy_down < min_avg) {
 338                 if (dev_priv->cur_delay != dev_priv->min_delay)
 339                         new_delay = dev_priv->cur_delay + 1;
 340                 if (new_delay > dev_priv->min_delay)
 341                         new_delay = dev_priv->min_delay;
 342         }
 343
 344         if (ironlake_set_drps(dev, new_delay))
 345                 dev_priv->cur_delay = new_delay;
 346
 347         return;
 348 }
 349
 350 static void notify_ring(struct drm_device *dev,
 351                         struct intel_ring_buffer *ring)
 352 {
 353         struct drm_i915_private *dev_priv = dev->dev_private;
 354         u32 seqno;
 355
 356         if (ring->obj == NULL)
 357                 return;
 358
 359         seqno = ring->get_seqno(ring);
 360         trace_i915_gem_request_complete(ring, seqno);
 361
 362         ring->irq_seqno = seqno;
 363         wake_up_all(&ring->irq_queue);
 364
 365         dev_priv->hangcheck_count = 0;
 366         mod_timer(&dev_priv->hangcheck_timer,
 367                   jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
 368 }
 369
 370 static void gen6_pm_irq_handler(struct drm_device *dev)
 371 {
 372         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 373         u8 new_delay = dev_priv->cur_delay;
 374         u32 pm_iir;
 375
 376         pm_iir = I915_READ(GEN6_PMIIR);
 377         if (!pm_iir)
 378                 return;
 379
 380         if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
 381                 if (dev_priv->cur_delay != dev_priv->max_delay)
 382                         new_delay = dev_priv->cur_delay + 1;
 383                 if (new_delay > dev_priv->max_delay)
 384                         new_delay = dev_priv->max_delay;
 385         } else if (pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT)) {
 386                 if (dev_priv->cur_delay != dev_priv->min_delay)
 387                         new_delay = dev_priv->cur_delay - 1;
 388                 if (new_delay < dev_priv->min_delay) {
 389                         new_delay = dev_priv->min_delay;
 390                         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
 391                                    I915_READ(GEN6_RP_INTERRUPT_LIMITS) |
 392                                    ((new_delay << 16) & 0x3f0000));
 393                 } else {
 394                         /* Make sure we continue to get down interrupts
 395                          * until we hit the minimum frequency */
 396                         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
 397                                    I915_READ(GEN6_RP_INTERRUPT_LIMITS) & ~0x3f0000);
 398                 }
 399         }
 400
 401         gen6_set_rps(dev, new_delay);
 402         dev_priv->cur_delay = new_delay;
 403
 404         I915_WRITE(GEN6_PMIIR, pm_iir);
 405 }
 406
 407 static void pch_irq_handler(struct drm_device *dev)
 408 {
 409         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 410         u32 pch_iir;
 411         int pipe;
 412
 413         pch_iir = I915_READ(SDEIIR);
 414
 415         if (pch_iir & SDE_AUDIO_POWER_MASK)
 416                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
 417                                  (pch_iir & SDE_AUDIO_POWER_MASK) >>
 418                                  SDE_AUDIO_POWER_SHIFT);
 419
 420         if (pch_iir & SDE_GMBUS)
 421                 DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n");
 422
 423         if (pch_iir & SDE_AUDIO_HDCP_MASK)
 424                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
 425
 426         if (pch_iir & SDE_AUDIO_TRANS_MASK)
 427                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
 428
 429         if (pch_iir & SDE_POISON)
 430                 DRM_ERROR("PCH poison interrupt\n");
 431
 432         if (pch_iir & SDE_FDI_MASK)
 433                 for_each_pipe(pipe)
 434                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
 435                                          pipe_name(pipe),
 436                                          I915_READ(FDI_RX_IIR(pipe)));
 437
 438         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
 439                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
 440
 441         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
 442                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
 443
 444         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
 445                 DRM_DEBUG_DRIVER("PCH transcoder B underrun interrupt\n");
 446         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
 447                 DRM_DEBUG_DRIVER("PCH transcoder A underrun interrupt\n");
 448 }
 449
 450 static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
 451 {
 452         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 453         int ret = IRQ_NONE;
 454         u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
 455         u32 hotplug_mask;
 456         struct drm_i915_master_private *master_priv;
 457         u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT;
 458
 459         if (IS_GEN6(dev))
 460                 bsd_usr_interrupt = GT_GEN6_BSD_USER_INTERRUPT;
 461
 462         /* disable master interrupt before clearing iir  */
 463         de_ier = I915_READ(DEIER);
 464         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
 465         POSTING_READ(DEIER);
 466
 467         de_iir = I915_READ(DEIIR);
 468         gt_iir = I915_READ(GTIIR);
 469         pch_iir = I915_READ(SDEIIR);
 470         pm_iir = I915_READ(GEN6_PMIIR);
 471
 472         if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 &&
 473             (!IS_GEN6(dev) || pm_iir == 0))
 474                 goto done;
 475
 476         if (HAS_PCH_CPT(dev))
 477                 hotplug_mask = SDE_HOTPLUG_MASK_CPT;
 478         else
 479                 hotplug_mask = SDE_HOTPLUG_MASK;
 480
 481         ret = IRQ_HANDLED;
 482
 483         if (dev->primary->master) {
 484                 master_priv = dev->primary->master->driver_priv;
 485                 if (master_priv->sarea_priv)
 486                         master_priv->sarea_priv->last_dispatch =
 487                                 READ_BREADCRUMB(dev_priv);
 488         }
 489
 490         if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
 491                 notify_ring(dev, &dev_priv->ring[RCS]);
 492         if (gt_iir & bsd_usr_interrupt)
 493                 notify_ring(dev, &dev_priv->ring[VCS]);
 494         if (gt_iir & GT_BLT_USER_INTERRUPT)
 495                 notify_ring(dev, &dev_priv->ring[BCS]);
 496
 497         if (de_iir & DE_GSE)
 498                 intel_opregion_gse_intr(dev);
 499
 500         if (de_iir & DE_PLANEA_FLIP_DONE) {
 501                 intel_prepare_page_flip(dev, 0);
 502                 intel_finish_page_flip_plane(dev, 0);
 503         }
 504
 505         if (de_iir & DE_PLANEB_FLIP_DONE) {
 506                 intel_prepare_page_flip(dev, 1);
 507                 intel_finish_page_flip_plane(dev, 1);
 508         }
 509
 510         if (de_iir & DE_PIPEA_VBLANK)
 511                 drm_handle_vblank(dev, 0);
 512
 513         if (de_iir & DE_PIPEB_VBLANK)
 514                 drm_handle_vblank(dev, 1);
 515
 516         /* check event from PCH */
 517         if (de_iir & DE_PCH_EVENT) {
 518                 if (pch_iir & hotplug_mask)
 519                         queue_work(dev_priv->wq, &dev_priv->hotplug_work);
 520                 pch_irq_handler(dev);
 521         }
 522
 523         if (de_iir & DE_PCU_EVENT) {
 524                 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 525                 i915_handle_rps_change(dev);
 526         }
 527
 528         if (IS_GEN6(dev))
 529                 gen6_pm_irq_handler(dev);
 530
 531         /* should clear PCH hotplug event before clear CPU irq */
 532         I915_WRITE(SDEIIR, pch_iir);
 533         I915_WRITE(GTIIR, gt_iir);
 534         I915_WRITE(DEIIR, de_iir);
 535
 536 done:
 537         I915_WRITE(DEIER, de_ier);
 538         POSTING_READ(DEIER);
 539
 540         return ret;
 541 }
 542
 543 /**
 544  * i915_error_work_func - do process context error handling work
 545  * @work: work struct
 546  *
 547  * Fire an error uevent so userspace can see that a hang or error
 548  * was detected.
 549  */
 550 static void i915_error_work_func(struct work_struct *work)
 551 {
 552         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 553                                                     error_work);
 554         struct drm_device *dev = dev_priv->dev;
 555         char *error_event[] = { "ERROR=1", NULL };
 556         char *reset_event[] = { "RESET=1", NULL };
 557         char *reset_done_event[] = { "ERROR=0", NULL };
 558
 559         kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
 560
 561         if (atomic_read(&dev_priv->mm.wedged)) {
 562                 DRM_DEBUG_DRIVER("resetting chip\n");
 563                 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event);
 564                 if (!i915_reset(dev, GRDOM_RENDER)) {
 565                         atomic_set(&dev_priv->mm.wedged, 0);
 566                         kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event);
 567                 }
 568                 complete_all(&dev_priv->error_completion);
 569         }
 570 }
 571
 572 #ifdef CONFIG_DEBUG_FS
 573 static struct drm_i915_error_object *
 574 i915_error_object_create(struct drm_i915_private *dev_priv,
 575                          struct drm_i915_gem_object *src)
 576 {
 577         struct drm_i915_error_object *dst;
 578         int page, page_count;
 579         u32 reloc_offset;
 580
 581         if (src == NULL || src->pages == NULL)
 582                 return NULL;
 583
 584         page_count = src->base.size / PAGE_SIZE;
 585
 586         dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC);
 587         if (dst == NULL)
 588                 return NULL;
 589
 590         reloc_offset = src->gtt_offset;
 591         for (page = 0; page < page_count; page++) {
 592                 unsigned long flags;
 593                 void __iomem *s;
 594                 void *d;
 595
 596                 d = kmalloc(PAGE_SIZE, GFP_ATOMIC);
 597                 if (d == NULL)
 598                         goto unwind;
 599
 600                 local_irq_save(flags);
 601                 s = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
 602                                              reloc_offset);
 603                 memcpy_fromio(d, s, PAGE_SIZE);
 604                 io_mapping_unmap_atomic(s);
 605                 local_irq_restore(flags);
 606
 607                 dst->pages[page] = d;
 608
 609                 reloc_offset += PAGE_SIZE;
 610         }
 611         dst->page_count = page_count;
 612         dst->gtt_offset = src->gtt_offset;
 613
 614         return dst;
 615
 616 unwind:
 617         while (page--)
 618                 kfree(dst->pages[page]);
 619         kfree(dst);
 620         return NULL;
 621 }
 622
 623 static void
 624 i915_error_object_free(struct drm_i915_error_object *obj)
 625 {
 626         int page;
 627
 628         if (obj == NULL)
 629                 return;
 630
 631         for (page = 0; page < obj->page_count; page++)
 632                 kfree(obj->pages[page]);
 633
 634         kfree(obj);
 635 }
 636
 637 static void
 638 i915_error_state_free(struct drm_device *dev,
 639                       struct drm_i915_error_state *error)
 640 {
 641         int i;
 642
 643         for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++)
 644                 i915_error_object_free(error->batchbuffer[i]);
 645
 646         for (i = 0; i < ARRAY_SIZE(error->ringbuffer); i++)
 647                 i915_error_object_free(error->ringbuffer[i]);
 648
 649         kfree(error->active_bo);
 650         kfree(error->overlay);
 651         kfree(error);
 652 }
 653
 654 static u32 capture_bo_list(struct drm_i915_error_buffer *err,
 655                            int count,
 656                            struct list_head *head)
 657 {
 658         struct drm_i915_gem_object *obj;
 659         int i = 0;
 660
 661         list_for_each_entry(obj, head, mm_list) {
 662                 err->size = obj->base.size;
 663                 err->name = obj->base.name;
 664                 err->seqno = obj->last_rendering_seqno;
 665                 err->gtt_offset = obj->gtt_offset;
 666                 err->read_domains = obj->base.read_domains;
 667                 err->write_domain = obj->base.write_domain;
 668                 err->fence_reg = obj->fence_reg;
 669                 err->pinned = 0;
 670                 if (obj->pin_count > 0)
 671                         err->pinned = 1;
 672                 if (obj->user_pin_count > 0)
 673                         err->pinned = -1;
 674                 err->tiling = obj->tiling_mode;
 675                 err->dirty = obj->dirty;
 676                 err->purgeable = obj->madv != I915_MADV_WILLNEED;
 677                 err->ring = obj->ring ? obj->ring->id : 0;
 678                 err->cache_level = obj->cache_level;
 679
 680                 if (++i == count)
 681                         break;
 682
 683                 err++;
 684         }
 685
 686         return i;
 687 }
 688
 689 static void i915_gem_record_fences(struct drm_device *dev,
 690                                    struct drm_i915_error_state *error)
 691 {
 692         struct drm_i915_private *dev_priv = dev->dev_private;
 693         int i;
 694
 695         /* Fences */
 696         switch (INTEL_INFO(dev)->gen) {
 697         case 6:
 698                 for (i = 0; i < 16; i++)
 699                         error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
 700                 break;
 701         case 5:
 702         case 4:
 703                 for (i = 0; i < 16; i++)
 704                         error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
 705                 break;
 706         case 3:
 707                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
 708                         for (i = 0; i < 8; i++)
 709                                 error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
 710         case 2:
 711                 for (i = 0; i < 8; i++)
 712                         error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
 713                 break;
 714
 715         }
 716 }
 717
 718 static struct drm_i915_error_object *
 719 i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
 720                              struct intel_ring_buffer *ring)
 721 {
 722         struct drm_i915_gem_object *obj;
 723         u32 seqno;
 724
 725         if (!ring->get_seqno)
 726                 return NULL;
 727
 728         seqno = ring->get_seqno(ring);
 729         list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
 730                 if (obj->ring != ring)
 731                         continue;
 732
 733                 if (i915_seqno_passed(seqno, obj->last_rendering_seqno))
 734                         continue;
 735
 736                 if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
 737                         continue;
 738
 739                 /* We need to copy these to an anonymous buffer as the simplest
 740                  * method to avoid being overwritten by userspace.
 741                  */
 742                 return i915_error_object_create(dev_priv, obj);
 743         }
 744
 745         return NULL;
 746 }
 747
 748 /**
 749  * i915_capture_error_state - capture an error record for later analysis
 750  * @dev: drm device
 751  *
 752  * Should be called when an error is detected (either a hang or an error
 753  * interrupt) to capture error state from the time of the error.  Fills
 754  * out a structure which becomes available in debugfs for user level tools
 755  * to pick up.
 756  */
 757 static void i915_capture_error_state(struct drm_device *dev)
 758 {
 759         struct drm_i915_private *dev_priv = dev->dev_private;
 760         struct drm_i915_gem_object *obj;
 761         struct drm_i915_error_state *error;
 762         unsigned long flags;
 763         int i, pipe;
 764
 765         spin_lock_irqsave(&dev_priv->error_lock, flags);
 766         error = dev_priv->first_error;
 767         spin_unlock_irqrestore(&dev_priv->error_lock, flags);
 768         if (error)
 769                 return;
 770
 771         /* Account for pipe specific data like PIPE*STAT */
 772         error = kmalloc(sizeof(*error), GFP_ATOMIC);
 773         if (!error) {
 774                 DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
 775                 return;
 776         }
 777
 778         DRM_INFO("capturing error event; look for more information in /debug/dri/%d/i915_error_state\n",
 779                  dev->primary->index);
 780
 781         error->seqno = dev_priv->ring[RCS].get_seqno(&dev_priv->ring[RCS]);
 782         error->eir = I915_READ(EIR);
 783         error->pgtbl_er = I915_READ(PGTBL_ER);
 784         for_each_pipe(pipe)
 785                 error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
 786         error->instpm = I915_READ(INSTPM);
 787         error->error = 0;
 788         if (INTEL_INFO(dev)->gen >= 6) {
 789                 error->error = I915_READ(ERROR_GEN6);
 790
 791                 error->bcs_acthd = I915_READ(BCS_ACTHD);
 792                 error->bcs_ipehr = I915_READ(BCS_IPEHR);
 793                 error->bcs_ipeir = I915_READ(BCS_IPEIR);
 794                 error->bcs_instdone = I915_READ(BCS_INSTDONE);
 795                 error->bcs_seqno = 0;
 796                 if (dev_priv->ring[BCS].get_seqno)
 797                         error->bcs_seqno = dev_priv->ring[BCS].get_seqno(&dev_priv->ring[BCS]);
 798
 799                 error->vcs_acthd = I915_READ(VCS_ACTHD);
 800                 error->vcs_ipehr = I915_READ(VCS_IPEHR);
 801                 error->vcs_ipeir = I915_READ(VCS_IPEIR);
 802                 error->vcs_instdone = I915_READ(VCS_INSTDONE);
 803                 error->vcs_seqno = 0;
 804                 if (dev_priv->ring[VCS].get_seqno)
 805                         error->vcs_seqno = dev_priv->ring[VCS].get_seqno(&dev_priv->ring[VCS]);
 806         }
 807         if (INTEL_INFO(dev)->gen >= 4) {
 808                 error->ipeir = I915_READ(IPEIR_I965);
 809                 error->ipehr = I915_READ(IPEHR_I965);
 810                 error->instdone = I915_READ(INSTDONE_I965);
 811                 error->instps = I915_READ(INSTPS);
 812                 error->instdone1 = I915_READ(INSTDONE1);
 813                 error->acthd = I915_READ(ACTHD_I965);
 814                 error->bbaddr = I915_READ64(BB_ADDR);
 815         } else {
 816                 error->ipeir = I915_READ(IPEIR);
 817                 error->ipehr = I915_READ(IPEHR);
 818                 error->instdone = I915_READ(INSTDONE);
 819                 error->acthd = I915_READ(ACTHD);
 820                 error->bbaddr = 0;
 821         }
 822         i915_gem_record_fences(dev, error);
 823
 824         /* Record the active batch and ring buffers */
 825         for (i = 0; i < I915_NUM_RINGS; i++) {
 826                 error->batchbuffer[i] =
 827                         i915_error_first_batchbuffer(dev_priv,
 828                                                      &dev_priv->ring[i]);
 829
 830                 error->ringbuffer[i] =
 831                         i915_error_object_create(dev_priv,
 832                                                  dev_priv->ring[i].obj);
 833         }
 834
 835         /* Record buffers on the active and pinned lists. */
 836         error->active_bo = NULL;
 837         error->pinned_bo = NULL;
 838
 839         i = 0;
 840         list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
 841                 i++;
 842         error->active_bo_count = i;
 843         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 844                 i++;
 845         error->pinned_bo_count = i - error->active_bo_count;
 846
 847         error->active_bo = NULL;
 848         error->pinned_bo = NULL;
 849         if (i) {
 850                 error->active_bo = kmalloc(sizeof(*error->active_bo)*i,
 851                                            GFP_ATOMIC);
 852                 if (error->active_bo)
 853                         error->pinned_bo =
 854                                 error->active_bo + error->active_bo_count;
 855         }
 856
 857         if (error->active_bo)
 858                 error->active_bo_count =
 859                         capture_bo_list(error->active_bo,
 860                                         error->active_bo_count,
 861                                         &dev_priv->mm.active_list);
 862
 863         if (error->pinned_bo)
 864                 error->pinned_bo_count =
 865                         capture_bo_list(error->pinned_bo,
 866                                         error->pinned_bo_count,
 867                                         &dev_priv->mm.pinned_list);
 868
 869         do_gettimeofday(&error->time);
 870
 871         error->overlay = intel_overlay_capture_error_state(dev);
 872         error->display = intel_display_capture_error_state(dev);
 873
 874         spin_lock_irqsave(&dev_priv->error_lock, flags);
 875         if (dev_priv->first_error == NULL) {
 876                 dev_priv->first_error = error;
 877                 error = NULL;
 878         }
 879         spin_unlock_irqrestore(&dev_priv->error_lock, flags);
 880
 881         if (error)
 882                 i915_error_state_free(dev, error);
 883 }
 884
 885 void i915_destroy_error_state(struct drm_device *dev)
 886 {
 887         struct drm_i915_private *dev_priv = dev->dev_private;
 888         struct drm_i915_error_state *error;
 889
 890         spin_lock(&dev_priv->error_lock);
 891         error = dev_priv->first_error;
 892         dev_priv->first_error = NULL;
 893         spin_unlock(&dev_priv->error_lock);
 894
 895         if (error)
 896                 i915_error_state_free(dev, error);
 897 }
 898 #else
 899 #define i915_capture_error_state(x)
 900 #endif
 901
 902 static void i915_report_and_clear_eir(struct drm_device *dev)
 903 {
 904         struct drm_i915_private *dev_priv = dev->dev_private;
 905         u32 eir = I915_READ(EIR);
 906         int pipe;
 907
 908         if (!eir)
 909                 return;
 910
 911         printk(KERN_ERR "render error detected, EIR: 0x%08x\n",
 912                eir);
 913
 914         if (IS_G4X(dev)) {
 915                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
 916                         u32 ipeir = I915_READ(IPEIR_I965);
 917
 918                         printk(KERN_ERR "  IPEIR: 0x%08x\n",
 919                                I915_READ(IPEIR_I965));
 920                         printk(KERN_ERR "  IPEHR: 0x%08x\n",
 921                                I915_READ(IPEHR_I965));
 922                         printk(KERN_ERR "  INSTDONE: 0x%08x\n",
 923                                I915_READ(INSTDONE_I965));
 924                         printk(KERN_ERR "  INSTPS: 0x%08x\n",
 925                                I915_READ(INSTPS));
 926                         printk(KERN_ERR "  INSTDONE1: 0x%08x\n",
 927                                I915_READ(INSTDONE1));
 928                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
 929                                I915_READ(ACTHD_I965));
 930                         I915_WRITE(IPEIR_I965, ipeir);
 931                         POSTING_READ(IPEIR_I965);
 932                 }
 933                 if (eir & GM45_ERROR_PAGE_TABLE) {
 934                         u32 pgtbl_err = I915_READ(PGTBL_ER);
 935                         printk(KERN_ERR "page table error\n");
 936                         printk(KERN_ERR "  PGTBL_ER: 0x%08x\n",
 937                                pgtbl_err);
 938                         I915_WRITE(PGTBL_ER, pgtbl_err);
 939                         POSTING_READ(PGTBL_ER);
 940                 }
 941         }
 942
 943         if (!IS_GEN2(dev)) {
 944                 if (eir & I915_ERROR_PAGE_TABLE) {
 945                         u32 pgtbl_err = I915_READ(PGTBL_ER);
 946                         printk(KERN_ERR "page table error\n");
 947                         printk(KERN_ERR "  PGTBL_ER: 0x%08x\n",
 948                                pgtbl_err);
 949                         I915_WRITE(PGTBL_ER, pgtbl_err);
 950                         POSTING_READ(PGTBL_ER);
 951                 }
 952         }
 953
 954         if (eir & I915_ERROR_MEMORY_REFRESH) {
 955                 printk(KERN_ERR "memory refresh error:\n");
 956                 for_each_pipe(pipe)
 957                         printk(KERN_ERR "pipe %c stat: 0x%08x\n",
 958                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
 959                 /* pipestat has already been acked */
 960         }
 961         if (eir & I915_ERROR_INSTRUCTION) {
 962                 printk(KERN_ERR "instruction error\n");
 963                 printk(KERN_ERR "  INSTPM: 0x%08x\n",
 964                        I915_READ(INSTPM));
 965                 if (INTEL_INFO(dev)->gen < 4) {
 966                         u32 ipeir = I915_READ(IPEIR);
 967
 968                         printk(KERN_ERR "  IPEIR: 0x%08x\n",
 969                                I915_READ(IPEIR));
 970                         printk(KERN_ERR "  IPEHR: 0x%08x\n",
 971                                I915_READ(IPEHR));
 972                         printk(KERN_ERR "  INSTDONE: 0x%08x\n",
 973                                I915_READ(INSTDONE));
 974                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
 975                                I915_READ(ACTHD));
 976                         I915_WRITE(IPEIR, ipeir);
 977                         POSTING_READ(IPEIR);
 978                 } else {
 979                         u32 ipeir = I915_READ(IPEIR_I965);
 980
 981                         printk(KERN_ERR "  IPEIR: 0x%08x\n",
 982                                I915_READ(IPEIR_I965));
 983                         printk(KERN_ERR "  IPEHR: 0x%08x\n",
 984                                I915_READ(IPEHR_I965));
 985                         printk(KERN_ERR "  INSTDONE: 0x%08x\n",
 986                                I915_READ(INSTDONE_I965));
 987                         printk(KERN_ERR "  INSTPS: 0x%08x\n",
 988                                I915_READ(INSTPS));
 989                         printk(KERN_ERR "  INSTDONE1: 0x%08x\n",
 990                                I915_READ(INSTDONE1));
 991                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
 992                                I915_READ(ACTHD_I965));
 993                         I915_WRITE(IPEIR_I965, ipeir);
 994                         POSTING_READ(IPEIR_I965);
 995                 }
 996         }
 997
 998         I915_WRITE(EIR, eir);
 999         POSTING_READ(EIR);
1000         eir = I915_READ(EIR);
1001         if (eir) {
1002                 /*
1003                  * some errors might have become stuck,
1004                  * mask them.
1005                  */
1006                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
1007                 I915_WRITE(EMR, I915_READ(EMR) | eir);
1008                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
1009         }
1010 }
1011
1012 /**
1013  * i915_handle_error - handle an error interrupt
1014  * @dev: drm device
1015  *
1016  * Do some basic checking of regsiter state at error interrupt time and
1017  * dump it to the syslog.  Also call i915_capture_error_state() to make
1018  * sure we get a record and make it available in debugfs.  Fire a uevent
1019  * so userspace knows something bad happened (should trigger collection
1020  * of a ring dump etc.).
1021  */
1022 void i915_handle_error(struct drm_device *dev, bool wedged)
1023 {
1024         struct drm_i915_private *dev_priv = dev->dev_private;
1025
1026         i915_capture_error_state(dev);
1027         i915_report_and_clear_eir(dev);
1028
1029         if (wedged) {
1030                 INIT_COMPLETION(dev_priv->error_completion);
1031                 atomic_set(&dev_priv->mm.wedged, 1);
1032
1033                 /*
1034                  * Wakeup waiting processes so they don't hang
1035                  */
1036                 wake_up_all(&dev_priv->ring[RCS].irq_queue);
1037                 if (HAS_BSD(dev))
1038                         wake_up_all(&dev_priv->ring[VCS].irq_queue);
1039                 if (HAS_BLT(dev))
1040                         wake_up_all(&dev_priv->ring[BCS].irq_queue);
1041         }
1042
1043         queue_work(dev_priv->wq, &dev_priv->error_work);
1044 }
1045
1046 static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
1047 {
1048         drm_i915_private_t *dev_priv = dev->dev_private;
1049         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1050         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1051         struct drm_i915_gem_object *obj;
1052         struct intel_unpin_work *work;
1053         unsigned long flags;
1054         bool stall_detected;
1055
1056         /* Ignore early vblank irqs */
1057         if (intel_crtc == NULL)
1058                 return;
1059
1060         spin_lock_irqsave(&dev->event_lock, flags);
1061         work = intel_crtc->unpin_work;
1062
1063         if (work == NULL || work->pending || !work->enable_stall_check) {
1064                 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
1065                 spin_unlock_irqrestore(&dev->event_lock, flags);
1066                 return;
1067         }
1068
1069         /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
1070         obj = work->pending_flip_obj;
1071         if (INTEL_INFO(dev)->gen >= 4) {
1072                 int dspsurf = DSPSURF(intel_crtc->plane);
1073                 stall_detected = I915_READ(dspsurf) == obj->gtt_offset;
1074         } else {
1075                 int dspaddr = DSPADDR(intel_crtc->plane);
1076                 stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
1077                                                         crtc->y * crtc->fb->pitch +
1078                                                         crtc->x * crtc->fb->bits_per_pixel/8);
1079         }
1080
1081         spin_unlock_irqrestore(&dev->event_lock, flags);
1082
1083         if (stall_detected) {
1084                 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
1085                 intel_prepare_page_flip(dev, intel_crtc->plane);
1086         }
1087 }
1088
1089 irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
1090 {
1091         struct drm_device *dev = (struct drm_device *) arg;
1092         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1093         struct drm_i915_master_private *master_priv;
1094         u32 iir, new_iir;
1095         u32 pipe_stats[I915_MAX_PIPES];
1096         u32 vblank_status;
1097         int vblank = 0;
1098         unsigned long irqflags;
1099         int irq_received;
1100         int ret = IRQ_NONE, pipe;
1101         bool blc_event = false;
1102
1103         atomic_inc(&dev_priv->irq_received);
1104
1105         if (HAS_PCH_SPLIT(dev))
1106                 return ironlake_irq_handler(dev);
1107
1108         iir = I915_READ(IIR);
1109
1110         if (INTEL_INFO(dev)->gen >= 4)
1111                 vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS;
1112         else
1113                 vblank_status = PIPE_VBLANK_INTERRUPT_STATUS;
1114
1115         for (;;) {
1116                 irq_received = iir != 0;
1117
1118                 /* Can't rely on pipestat interrupt bit in iir as it might
1119                  * have been cleared after the pipestat interrupt was received.
1120                  * It doesn't set the bit in iir again, but it still produces
1121                  * interrupts (for non-MSI).
1122                  */
1123                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1124                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
1125                         i915_handle_error(dev, false);
1126
1127                 for_each_pipe(pipe) {
1128                         int reg = PIPESTAT(pipe);
1129                         pipe_stats[pipe] = I915_READ(reg);
1130
1131                         /*
1132                          * Clear the PIPE*STAT regs before the IIR
1133                          */
1134                         if (pipe_stats[pipe] & 0x8000ffff) {
1135                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1136                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
1137                                                          pipe_name(pipe));
1138                                 I915_WRITE(reg, pipe_stats[pipe]);
1139                                 irq_received = 1;
1140                         }
1141                 }
1142                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1143
1144                 if (!irq_received)
1145                         break;
1146
1147                 ret = IRQ_HANDLED;
1148
1149                 /* Consume port.  Then clear IIR or we'll miss events */
1150                 if ((I915_HAS_HOTPLUG(dev)) &&
1151                     (iir & I915_DISPLAY_PORT_INTERRUPT)) {
1152                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1153
1154                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1155                                   hotplug_status);
1156                         if (hotplug_status & dev_priv->hotplug_supported_mask)
1157                                 queue_work(dev_priv->wq,
1158                                            &dev_priv->hotplug_work);
1159
1160                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1161                         I915_READ(PORT_HOTPLUG_STAT);
1162                 }
1163
1164                 I915_WRITE(IIR, iir);
1165                 new_iir = I915_READ(IIR); /* Flush posted writes */
1166
1167                 if (dev->primary->master) {
1168                         master_priv = dev->primary->master->driver_priv;
1169                         if (master_priv->sarea_priv)
1170                                 master_priv->sarea_priv->last_dispatch =
1171                                         READ_BREADCRUMB(dev_priv);
1172                 }
1173
1174                 if (iir & I915_USER_INTERRUPT)
1175                         notify_ring(dev, &dev_priv->ring[RCS]);
1176                 if (iir & I915_BSD_USER_INTERRUPT)
1177                         notify_ring(dev, &dev_priv->ring[VCS]);
1178
1179                 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
1180                         intel_prepare_page_flip(dev, 0);
1181                         if (dev_priv->flip_pending_is_done)
1182                                 intel_finish_page_flip_plane(dev, 0);
1183                 }
1184
1185                 if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT) {
1186                         intel_prepare_page_flip(dev, 1);
1187                         if (dev_priv->flip_pending_is_done)
1188                                 intel_finish_page_flip_plane(dev, 1);
1189                 }
1190
1191                 for_each_pipe(pipe) {
1192                         if (pipe_stats[pipe] & vblank_status &&
1193                             drm_handle_vblank(dev, pipe)) {
1194                                 vblank++;
1195                                 if (!dev_priv->flip_pending_is_done) {
1196                                         i915_pageflip_stall_check(dev, pipe);
1197                                         intel_finish_page_flip(dev, pipe);
1198                                 }
1199                         }
1200
1201                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1202                                 blc_event = true;
1203                 }
1204
1205
1206                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
1207                         intel_opregion_asle_intr(dev);
1208
1209                 /* With MSI, interrupts are only generated when iir
1210                  * transitions from zero to nonzero.  If another bit got
1211                  * set while we were handling the existing iir bits, then
1212                  * we would never get another interrupt.
1213                  *
1214                  * This is fine on non-MSI as well, as if we hit this path
1215                  * we avoid exiting the interrupt handler only to generate
1216                  * another one.
1217                  *
1218                  * Note that for MSI this could cause a stray interrupt report
1219                  * if an interrupt landed in the time between writing IIR and
1220                  * the posting read.  This should be rare enough to never
1221                  * trigger the 99% of 100,000 interrupts test for disabling
1222                  * stray interrupts.
1223                  */
1224                 iir = new_iir;
1225         }
1226
1227         return ret;
1228 }
1229
1230 static int i915_emit_irq(struct drm_device * dev)
1231 {
1232         drm_i915_private_t *dev_priv = dev->dev_private;
1233         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1234
1235         i915_kernel_lost_context(dev);
1236
1237         DRM_DEBUG_DRIVER("\n");
1238
1239         dev_priv->counter++;
1240         if (dev_priv->counter > 0x7FFFFFFFUL)
1241                 dev_priv->counter = 1;
1242         if (master_priv->sarea_priv)
1243                 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
1244
1245         if (BEGIN_LP_RING(4) == 0) {
1246                 OUT_RING(MI_STORE_DWORD_INDEX);
1247                 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1248                 OUT_RING(dev_priv->counter);
1249                 OUT_RING(MI_USER_INTERRUPT);
1250                 ADVANCE_LP_RING();
1251         }
1252
1253         return dev_priv->counter;
1254 }
1255
1256 static int i915_wait_irq(struct drm_device * dev, int irq_nr)
1257 {
1258         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1259         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1260         int ret = 0;
1261         struct intel_ring_buffer *ring = LP_RING(dev_priv);
1262
1263         DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr,
1264                   READ_BREADCRUMB(dev_priv));
1265
1266         if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
1267                 if (master_priv->sarea_priv)
1268                         master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
1269                 return 0;
1270         }
1271
1272         if (master_priv->sarea_priv)
1273                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1274
1275         if (ring->irq_get(ring)) {
1276                 DRM_WAIT_ON(ret, ring->irq_queue, 3 * DRM_HZ,
1277                             READ_BREADCRUMB(dev_priv) >= irq_nr);
1278                 ring->irq_put(ring);
1279         } else if (wait_for(READ_BREADCRUMB(dev_priv) >= irq_nr, 3000))
1280                 ret = -EBUSY;
1281
1282         if (ret == -EBUSY) {
1283                 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
1284                           READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
1285         }
1286
1287         return ret;
1288 }
1289
1290 /* Needs the lock as it touches the ring.
1291  */
1292 int i915_irq_emit(struct drm_device *dev, void *data,
1293                          struct drm_file *file_priv)
1294 {
1295         drm_i915_private_t *dev_priv = dev->dev_private;
1296         drm_i915_irq_emit_t *emit = data;
1297         int result;
1298
1299         if (!dev_priv || !LP_RING(dev_priv)->virtual_start) {
1300                 DRM_ERROR("called with no initialization\n");
1301                 return -EINVAL;
1302         }
1303
1304         RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
1305
1306         mutex_lock(&dev->struct_mutex);
1307         result = i915_emit_irq(dev);
1308         mutex_unlock(&dev->struct_mutex);
1309
1310         if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
1311                 DRM_ERROR("copy_to_user\n");
1312                 return -EFAULT;
1313         }
1314
1315         return 0;
1316 }
1317
1318 /* Doesn't need the hardware lock.
1319  */
1320 int i915_irq_wait(struct drm_device *dev, void *data,
1321                          struct drm_file *file_priv)
1322 {
1323         drm_i915_private_t *dev_priv = dev->dev_private;
1324         drm_i915_irq_wait_t *irqwait = data;
1325
1326         if (!dev_priv) {
1327                 DRM_ERROR("called with no initialization\n");
1328                 return -EINVAL;
1329         }
1330
1331         return i915_wait_irq(dev, irqwait->irq_seq);
1332 }
1333
1334 /* Called from drm generic code, passed 'crtc' which
1335  * we use as a pipe index
1336  */
1337 int i915_enable_vblank(struct drm_device *dev, int pipe)
1338 {
1339         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1340         unsigned long irqflags;
1341
1342         if (!i915_pipe_enabled(dev, pipe))
1343                 return -EINVAL;
1344
1345         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1346         if (HAS_PCH_SPLIT(dev))
1347                 ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
1348                                             DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
1349         else if (INTEL_INFO(dev)->gen >= 4)
1350                 i915_enable_pipestat(dev_priv, pipe,
1351                                      PIPE_START_VBLANK_INTERRUPT_ENABLE);
1352         else
1353                 i915_enable_pipestat(dev_priv, pipe,
1354                                      PIPE_VBLANK_INTERRUPT_ENABLE);
1355
1356         /* maintain vblank delivery even in deep C-states */
1357         if (dev_priv->info->gen == 3)
1358                 I915_WRITE(INSTPM, INSTPM_AGPBUSY_DIS << 16);
1359         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1360
1361         return 0;
1362 }
1363
1364 /* Called from drm generic code, passed 'crtc' which
1365  * we use as a pipe index
1366  */
1367 void i915_disable_vblank(struct drm_device *dev, int pipe)
1368 {
1369         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1370         unsigned long irqflags;
1371
1372         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1373         if (dev_priv->info->gen == 3)
1374                 I915_WRITE(INSTPM,
1375                            INSTPM_AGPBUSY_DIS << 16 | INSTPM_AGPBUSY_DIS);
1376
1377         if (HAS_PCH_SPLIT(dev))
1378                 ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
1379                                              DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
1380         else
1381                 i915_disable_pipestat(dev_priv, pipe,
1382                                       PIPE_VBLANK_INTERRUPT_ENABLE |
1383                                       PIPE_START_VBLANK_INTERRUPT_ENABLE);
1384         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1385 }
1386
1387 /* Set the vblank monitor pipe
1388  */
1389 int i915_vblank_pipe_set(struct drm_device *dev, void *data,
1390                          struct drm_file *file_priv)
1391 {
1392         drm_i915_private_t *dev_priv = dev->dev_private;
1393
1394         if (!dev_priv) {
1395                 DRM_ERROR("called with no initialization\n");
1396                 return -EINVAL;
1397         }
1398
1399         return 0;
1400 }
1401
1402 int i915_vblank_pipe_get(struct drm_device *dev, void *data,
1403                          struct drm_file *file_priv)
1404 {
1405         drm_i915_private_t *dev_priv = dev->dev_private;
1406         drm_i915_vblank_pipe_t *pipe = data;
1407
1408         if (!dev_priv) {
1409                 DRM_ERROR("called with no initialization\n");
1410                 return -EINVAL;
1411         }
1412
1413         pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
1414
1415         return 0;
1416 }
1417
1418 /**
1419  * Schedule buffer swap at given vertical blank.
1420  */
1421 int i915_vblank_swap(struct drm_device *dev, void *data,
1422                      struct drm_file *file_priv)
1423 {
1424         /* The delayed swap mechanism was fundamentally racy, and has been
1425          * removed.  The model was that the client requested a delayed flip/swap
1426          * from the kernel, then waited for vblank before continuing to perform
1427          * rendering.  The problem was that the kernel might wake the client
1428          * up before it dispatched the vblank swap (since the lock has to be
1429          * held while touching the ringbuffer), in which case the client would
1430          * clear and start the next frame before the swap occurred, and
1431          * flicker would occur in addition to likely missing the vblank.
1432          *
1433          * In the absence of this ioctl, userland falls back to a correct path
1434          * of waiting for a vblank, then dispatching the swap on its own.
1435          * Context switching to userland and back is plenty fast enough for
1436          * meeting the requirements of vblank swapping.
1437          */
1438         return -EINVAL;
1439 }
1440
1441 static u32
1442 ring_last_seqno(struct intel_ring_buffer *ring)
1443 {
1444         return list_entry(ring->request_list.prev,
1445                           struct drm_i915_gem_request, list)->seqno;
1446 }
1447
1448 static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
1449 {
1450         if (list_empty(&ring->request_list) ||
1451             i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) {
1452                 /* Issue a wake-up to catch stuck h/w. */
1453                 if (ring->waiting_seqno && waitqueue_active(&ring->irq_queue)) {
1454                         DRM_ERROR("Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n",
1455                                   ring->name,
1456                                   ring->waiting_seqno,
1457                                   ring->get_seqno(ring));
1458                         wake_up_all(&ring->irq_queue);
1459                         *err = true;
1460                 }
1461                 return true;
1462         }
1463         return false;
1464 }
1465
1466 static bool kick_ring(struct intel_ring_buffer *ring)
1467 {
1468         struct drm_device *dev = ring->dev;
1469         struct drm_i915_private *dev_priv = dev->dev_private;
1470         u32 tmp = I915_READ_CTL(ring);
1471         if (tmp & RING_WAIT) {
1472                 DRM_ERROR("Kicking stuck wait on %s\n",
1473                           ring->name);
1474                 I915_WRITE_CTL(ring, tmp);
1475                 return true;
1476         }
1477         if (IS_GEN6(dev) &&
1478             (tmp & RING_WAIT_SEMAPHORE)) {
1479                 DRM_ERROR("Kicking stuck semaphore on %s\n",
1480                           ring->name);
1481                 I915_WRITE_CTL(ring, tmp);
1482                 return true;
1483         }
1484         return false;
1485 }
1486
1487 /**
1488  * This is called when the chip hasn't reported back with completed
1489  * batchbuffers in a long time. The first time this is called we simply record
1490  * ACTHD. If ACTHD hasn't changed by the time the hangcheck timer elapses
1491  * again, we assume the chip is wedged and try to fix it.
1492  */
1493 void i915_hangcheck_elapsed(unsigned long data)
1494 {
1495         struct drm_device *dev = (struct drm_device *)data;
1496         drm_i915_private_t *dev_priv = dev->dev_private;
1497         uint32_t acthd, instdone, instdone1;
1498         bool err = false;
1499
1500         /* If all work is done then ACTHD clearly hasn't advanced. */
1501         if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) &&
1502             i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) &&
1503             i915_hangcheck_ring_idle(&dev_priv->ring[BCS], &err)) {
1504                 dev_priv->hangcheck_count = 0;
1505                 if (err)
1506                         goto repeat;
1507                 return;
1508         }
1509
1510         if (INTEL_INFO(dev)->gen < 4) {
1511                 acthd = I915_READ(ACTHD);
1512                 instdone = I915_READ(INSTDONE);
1513                 instdone1 = 0;
1514         } else {
1515                 acthd = I915_READ(ACTHD_I965);
1516                 instdone = I915_READ(INSTDONE_I965);
1517                 instdone1 = I915_READ(INSTDONE1);
1518         }
1519
1520         if (dev_priv->last_acthd == acthd &&
1521             dev_priv->last_instdone == instdone &&
1522             dev_priv->last_instdone1 == instdone1) {
1523                 if (dev_priv->hangcheck_count++ > 1) {
1524                         DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
1525
1526                         if (!IS_GEN2(dev)) {
1527                                 /* Is the chip hanging on a WAIT_FOR_EVENT?
1528                                  * If so we can simply poke the RB_WAIT bit
1529                                  * and break the hang. This should work on
1530                                  * all but the second generation chipsets.
1531                                  */
1532
1533                                 if (kick_ring(&dev_priv->ring[RCS]))
1534                                         goto repeat;
1535
1536                                 if (HAS_BSD(dev) &&
1537                                     kick_ring(&dev_priv->ring[VCS]))
1538                                         goto repeat;
1539
1540                                 if (HAS_BLT(dev) &&
1541                                     kick_ring(&dev_priv->ring[BCS]))
1542                                         goto repeat;
1543                         }
1544
1545                         i915_handle_error(dev, true);
1546                         return;
1547                 }
1548         } else {
1549                 dev_priv->hangcheck_count = 0;
1550
1551                 dev_priv->last_acthd = acthd;
1552                 dev_priv->last_instdone = instdone;
1553                 dev_priv->last_instdone1 = instdone1;
1554         }
1555
1556 repeat:
1557         /* Reset timer case chip hangs without another request being added */
1558         mod_timer(&dev_priv->hangcheck_timer,
1559                   jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1560 }
1561
1562 /* drm_dma.h hooks
1563 */
1564 static void ironlake_irq_preinstall(struct drm_device *dev)
1565 {
1566         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1567
1568         I915_WRITE(HWSTAM, 0xeffe);
1569
1570         /* XXX hotplug from PCH */
1571
1572         I915_WRITE(DEIMR, 0xffffffff);
1573         I915_WRITE(DEIER, 0x0);
1574         POSTING_READ(DEIER);
1575
1576         /* and GT */
1577         I915_WRITE(GTIMR, 0xffffffff);
1578         I915_WRITE(GTIER, 0x0);
1579         POSTING_READ(GTIER);
1580
1581         /* south display irq */
1582         I915_WRITE(SDEIMR, 0xffffffff);
1583         I915_WRITE(SDEIER, 0x0);
1584         POSTING_READ(SDEIER);
1585 }
1586
1587 static int ironlake_irq_postinstall(struct drm_device *dev)
1588 {
1589         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1590         /* enable kind of interrupts always enabled */
1591         u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
1592                            DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE;
1593         u32 render_irqs;
1594         u32 hotplug_mask;
1595
1596         dev_priv->irq_mask = ~display_mask;
1597
1598         /* should always can generate irq */
1599         I915_WRITE(DEIIR, I915_READ(DEIIR));
1600         I915_WRITE(DEIMR, dev_priv->irq_mask);
1601         I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
1602         POSTING_READ(DEIER);
1603
1604         dev_priv->gt_irq_mask = ~0;
1605
1606         I915_WRITE(GTIIR, I915_READ(GTIIR));
1607         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
1608
1609         if (IS_GEN6(dev))
1610                 render_irqs =
1611                         GT_USER_INTERRUPT |
1612                         GT_GEN6_BSD_USER_INTERRUPT |
1613                         GT_BLT_USER_INTERRUPT;
1614         else
1615                 render_irqs =
1616                         GT_USER_INTERRUPT |
1617                         GT_PIPE_NOTIFY |
1618                         GT_BSD_USER_INTERRUPT;
1619         I915_WRITE(GTIER, render_irqs);
1620         POSTING_READ(GTIER);
1621
1622         if (HAS_PCH_CPT(dev)) {
1623                 hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
1624                                 SDE_PORTB_HOTPLUG_CPT |
1625                                 SDE_PORTC_HOTPLUG_CPT |
1626                                 SDE_PORTD_HOTPLUG_CPT);
1627         } else {
1628                 hotplug_mask = (SDE_CRT_HOTPLUG |
1629                                 SDE_PORTB_HOTPLUG |
1630                                 SDE_PORTC_HOTPLUG |
1631                                 SDE_PORTD_HOTPLUG |
1632                                 SDE_AUX_MASK);
1633         }
1634
1635         dev_priv->pch_irq_mask = ~hotplug_mask;
1636
1637         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
1638         I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
1639         I915_WRITE(SDEIER, hotplug_mask);
1640         POSTING_READ(SDEIER);
1641
1642         if (IS_IRONLAKE_M(dev)) {
1643                 /* Clear & enable PCU event interrupts */
1644                 I915_WRITE(DEIIR, DE_PCU_EVENT);
1645                 I915_WRITE(DEIER, I915_READ(DEIER) | DE_PCU_EVENT);
1646                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
1647         }
1648
1649         return 0;
1650 }
1651
1652 void i915_driver_irq_preinstall(struct drm_device * dev)
1653 {
1654         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1655         int pipe;
1656
1657         atomic_set(&dev_priv->irq_received, 0);
1658
1659         INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
1660         INIT_WORK(&dev_priv->error_work, i915_error_work_func);
1661
1662         if (HAS_PCH_SPLIT(dev)) {
1663                 ironlake_irq_preinstall(dev);
1664                 return;
1665         }
1666
1667         if (I915_HAS_HOTPLUG(dev)) {
1668                 I915_WRITE(PORT_HOTPLUG_EN, 0);
1669                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
1670         }
1671
1672         I915_WRITE(HWSTAM, 0xeffe);
1673         for_each_pipe(pipe)
1674                 I915_WRITE(PIPESTAT(pipe), 0);
1675         I915_WRITE(IMR, 0xffffffff);
1676         I915_WRITE(IER, 0x0);
1677         POSTING_READ(IER);
1678 }
1679
1680 /*
1681  * Must be called after intel_modeset_init or hotplug interrupts won't be
1682  * enabled correctly.
1683  */
1684 int i915_driver_irq_postinstall(struct drm_device *dev)
1685 {
1686         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1687         u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR;
1688         u32 error_mask;
1689
1690         dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
1691
1692         if (HAS_PCH_SPLIT(dev))
1693                 return ironlake_irq_postinstall(dev);
1694
1695         /* Unmask the interrupts that we always want on. */
1696         dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX;
1697
1698         dev_priv->pipestat[0] = 0;
1699         dev_priv->pipestat[1] = 0;
1700
1701         if (I915_HAS_HOTPLUG(dev)) {
1702                 /* Enable in IER... */
1703                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
1704                 /* and unmask in IMR */
1705                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
1706         }
1707
1708         /*
1709          * Enable some error detection, note the instruction error mask
1710          * bit is reserved, so we leave it masked.
1711          */
1712         if (IS_G4X(dev)) {
1713                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
1714                                GM45_ERROR_MEM_PRIV |
1715                                GM45_ERROR_CP_PRIV |
1716                                I915_ERROR_MEMORY_REFRESH);
1717         } else {
1718                 error_mask = ~(I915_ERROR_PAGE_TABLE |
1719                                I915_ERROR_MEMORY_REFRESH);
1720         }
1721         I915_WRITE(EMR, error_mask);
1722
1723         I915_WRITE(IMR, dev_priv->irq_mask);
1724         I915_WRITE(IER, enable_mask);
1725         POSTING_READ(IER);
1726
1727         if (I915_HAS_HOTPLUG(dev)) {
1728                 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
1729
1730                 /* Note HDMI and DP share bits */
1731                 if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
1732                         hotplug_en |= HDMIB_HOTPLUG_INT_EN;
1733                 if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
1734                         hotplug_en |= HDMIC_HOTPLUG_INT_EN;
1735                 if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
1736                         hotplug_en |= HDMID_HOTPLUG_INT_EN;
1737                 if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS)
1738                         hotplug_en |= SDVOC_HOTPLUG_INT_EN;
1739                 if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS)
1740                         hotplug_en |= SDVOB_HOTPLUG_INT_EN;
1741                 if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) {
1742                         hotplug_en |= CRT_HOTPLUG_INT_EN;
1743
1744                         /* Programming the CRT detection parameters tends
1745                            to generate a spurious hotplug event about three
1746                            seconds later.  So just do it once.
1747                         */
1748                         if (IS_G4X(dev))
1749                                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
1750                         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
1751                 }
1752
1753                 /* Ignore TV since it's buggy */
1754
1755                 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
1756         }
1757
1758         intel_opregion_enable_asle(dev);
1759
1760         return 0;
1761 }
1762
1763 static void ironlake_irq_uninstall(struct drm_device *dev)
1764 {
1765         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1766         I915_WRITE(HWSTAM, 0xffffffff);
1767
1768         I915_WRITE(DEIMR, 0xffffffff);
1769         I915_WRITE(DEIER, 0x0);
1770         I915_WRITE(DEIIR, I915_READ(DEIIR));
1771
1772         I915_WRITE(GTIMR, 0xffffffff);
1773         I915_WRITE(GTIER, 0x0);
1774         I915_WRITE(GTIIR, I915_READ(GTIIR));
1775 }
1776
1777 void i915_driver_irq_uninstall(struct drm_device * dev)
1778 {
1779         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1780         int pipe;
1781
1782         if (!dev_priv)
1783                 return;
1784
1785         dev_priv->vblank_pipe = 0;
1786
1787         if (HAS_PCH_SPLIT(dev)) {
1788                 ironlake_irq_uninstall(dev);
1789                 return;
1790         }
1791
1792         if (I915_HAS_HOTPLUG(dev)) {
1793                 I915_WRITE(PORT_HOTPLUG_EN, 0);
1794                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
1795         }
1796
1797         I915_WRITE(HWSTAM, 0xffffffff);
1798         for_each_pipe(pipe)
1799                 I915_WRITE(PIPESTAT(pipe), 0);
1800         I915_WRITE(IMR, 0xffffffff);
1801         I915_WRITE(IER, 0x0);
1802
1803         for_each_pipe(pipe)
1804                 I915_WRITE(PIPESTAT(pipe),
1805                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
1806         I915_WRITE(IIR, I915_READ(IIR));
1807 }