drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <drm/drmP.h>
  34 #include <drm/i915_drm.h>
  35 #include "i915_drv.h"
  36 #include "i915_trace.h"
  37 #include "intel_drv.h"
  38
  39 static const u32 hpd_ibx[] = {
  40         [HPD_CRT] = SDE_CRT_HOTPLUG,
  41         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  42         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  43         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  44         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  45 };
  46
  47 static const u32 hpd_cpt[] = {
  48         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  49         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  50         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  51         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  52         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  53 };
  54
  55 static const u32 hpd_mask_i915[] = {
  56         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  57         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  58         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  59         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  60         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  61         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  62 };
  63
  64 static const u32 hpd_status_gen4[] = {
  65         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  66         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  67         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  68         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  69         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  70         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  71 };
  72
  73 static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
  74         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  75         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
  76         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
  77         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  78         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  79         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  80 };
  81
  82 /* For display hotplug interrupt */
  83 static void
  84 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  85 {
  86         assert_spin_locked(&dev_priv->irq_lock);
  87
  88         if ((dev_priv->irq_mask & mask) != 0) {
  89                 dev_priv->irq_mask &= ~mask;
  90                 I915_WRITE(DEIMR, dev_priv->irq_mask);
  91                 POSTING_READ(DEIMR);
  92         }
  93 }
  94
  95 static void
  96 ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  97 {
  98         assert_spin_locked(&dev_priv->irq_lock);
  99
 100         if ((dev_priv->irq_mask & mask) != mask) {
 101                 dev_priv->irq_mask |= mask;
 102                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 103                 POSTING_READ(DEIMR);
 104         }
 105 }
 106
 107 /**
 108  * ilk_update_gt_irq - update GTIMR
 109  * @dev_priv: driver private
 110  * @interrupt_mask: mask of interrupt bits to update
 111  * @enabled_irq_mask: mask of interrupt bits to enable
 112  */
 113 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 114                               uint32_t interrupt_mask,
 115                               uint32_t enabled_irq_mask)
 116 {
 117         assert_spin_locked(&dev_priv->irq_lock);
 118
 119         dev_priv->gt_irq_mask &= ~interrupt_mask;
 120         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
 121         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 122         POSTING_READ(GTIMR);
 123 }
 124
 125 void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 126 {
 127         ilk_update_gt_irq(dev_priv, mask, mask);
 128 }
 129
 130 void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 131 {
 132         ilk_update_gt_irq(dev_priv, mask, 0);
 133 }
 134
 135 /**
 136   * snb_update_pm_irq - update GEN6_PMIMR
 137   * @dev_priv: driver private
 138   * @interrupt_mask: mask of interrupt bits to update
 139   * @enabled_irq_mask: mask of interrupt bits to enable
 140   */
 141 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 142                               uint32_t interrupt_mask,
 143                               uint32_t enabled_irq_mask)
 144 {
 145         uint32_t new_val;
 146
 147         assert_spin_locked(&dev_priv->irq_lock);
 148
 149         new_val = dev_priv->pm_irq_mask;
 150         new_val &= ~interrupt_mask;
 151         new_val |= (~enabled_irq_mask & interrupt_mask);
 152
 153         if (new_val != dev_priv->pm_irq_mask) {
 154                 dev_priv->pm_irq_mask = new_val;
 155                 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
 156                 POSTING_READ(GEN6_PMIMR);
 157         }
 158 }
 159
 160 void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 161 {
 162         snb_update_pm_irq(dev_priv, mask, mask);
 163 }
 164
 165 void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 166 {
 167         snb_update_pm_irq(dev_priv, mask, 0);
 168 }
 169
 170 static void snb_set_pm_irq(struct drm_i915_private *dev_priv, uint32_t val)
 171 {
 172         snb_update_pm_irq(dev_priv, 0xffffffff, ~val);
 173 }
 174
 175 static bool ivb_can_enable_err_int(struct drm_device *dev)
 176 {
 177         struct drm_i915_private *dev_priv = dev->dev_private;
 178         struct intel_crtc *crtc;
 179         enum pipe pipe;
 180
 181         assert_spin_locked(&dev_priv->irq_lock);
 182
 183         for_each_pipe(pipe) {
 184                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 185
 186                 if (crtc->cpu_fifo_underrun_disabled)
 187                         return false;
 188         }
 189
 190         return true;
 191 }
 192
 193 static bool cpt_can_enable_serr_int(struct drm_device *dev)
 194 {
 195         struct drm_i915_private *dev_priv = dev->dev_private;
 196         enum pipe pipe;
 197         struct intel_crtc *crtc;
 198
 199         assert_spin_locked(&dev_priv->irq_lock);
 200
 201         for_each_pipe(pipe) {
 202                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 203
 204                 if (crtc->pch_fifo_underrun_disabled)
 205                         return false;
 206         }
 207
 208         return true;
 209 }
 210
 211 static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
 212                                                  enum pipe pipe, bool enable)
 213 {
 214         struct drm_i915_private *dev_priv = dev->dev_private;
 215         uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
 216                                           DE_PIPEB_FIFO_UNDERRUN;
 217
 218         if (enable)
 219                 ironlake_enable_display_irq(dev_priv, bit);
 220         else
 221                 ironlake_disable_display_irq(dev_priv, bit);
 222 }
 223
 224 static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
 225                                                   enum pipe pipe, bool enable)
 226 {
 227         struct drm_i915_private *dev_priv = dev->dev_private;
 228         if (enable) {
 229                 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
 230
 231                 if (!ivb_can_enable_err_int(dev))
 232                         return;
 233
 234                 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
 235         } else {
 236                 bool was_enabled = !(I915_READ(DEIMR) & DE_ERR_INT_IVB);
 237
 238                 /* Change the state _after_ we've read out the current one. */
 239                 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
 240
 241                 if (!was_enabled &&
 242                     (I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe))) {
 243                         DRM_DEBUG_KMS("uncleared fifo underrun on pipe %c\n",
 244                                       pipe_name(pipe));
 245                 }
 246         }
 247 }
 248
 249 /**
 250  * ibx_display_interrupt_update - update SDEIMR
 251  * @dev_priv: driver private
 252  * @interrupt_mask: mask of interrupt bits to update
 253  * @enabled_irq_mask: mask of interrupt bits to enable
 254  */
 255 static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 256                                          uint32_t interrupt_mask,
 257                                          uint32_t enabled_irq_mask)
 258 {
 259         uint32_t sdeimr = I915_READ(SDEIMR);
 260         sdeimr &= ~interrupt_mask;
 261         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 262
 263         assert_spin_locked(&dev_priv->irq_lock);
 264
 265         I915_WRITE(SDEIMR, sdeimr);
 266         POSTING_READ(SDEIMR);
 267 }
 268 #define ibx_enable_display_interrupt(dev_priv, bits) \
 269         ibx_display_interrupt_update((dev_priv), (bits), (bits))
 270 #define ibx_disable_display_interrupt(dev_priv, bits) \
 271         ibx_display_interrupt_update((dev_priv), (bits), 0)
 272
 273 static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
 274                                             enum transcoder pch_transcoder,
 275                                             bool enable)
 276 {
 277         struct drm_i915_private *dev_priv = dev->dev_private;
 278         uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
 279                        SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
 280
 281         if (enable)
 282                 ibx_enable_display_interrupt(dev_priv, bit);
 283         else
 284                 ibx_disable_display_interrupt(dev_priv, bit);
 285 }
 286
 287 static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
 288                                             enum transcoder pch_transcoder,
 289                                             bool enable)
 290 {
 291         struct drm_i915_private *dev_priv = dev->dev_private;
 292
 293         if (enable) {
 294                 I915_WRITE(SERR_INT,
 295                            SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));
 296
 297                 if (!cpt_can_enable_serr_int(dev))
 298                         return;
 299
 300                 ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
 301         } else {
 302                 uint32_t tmp = I915_READ(SERR_INT);
 303                 bool was_enabled = !(I915_READ(SDEIMR) & SDE_ERROR_CPT);
 304
 305                 /* Change the state _after_ we've read out the current one. */
 306                 ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
 307
 308                 if (!was_enabled &&
 309                     (tmp & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder))) {
 310                         DRM_DEBUG_KMS("uncleared pch fifo underrun on pch transcoder %c\n",
 311                                       transcoder_name(pch_transcoder));
 312                 }
 313         }
 314 }
 315
 316 /**
 317  * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
 318  * @dev: drm device
 319  * @pipe: pipe
 320  * @enable: true if we want to report FIFO underrun errors, false otherwise
 321  *
 322  * This function makes us disable or enable CPU fifo underruns for a specific
 323  * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
 324  * reporting for one pipe may also disable all the other CPU error interruts for
 325  * the other pipes, due to the fact that there's just one interrupt mask/enable
 326  * bit for all the pipes.
 327  *
 328  * Returns the previous state of underrun reporting.
 329  */
 330 bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
 331                                            enum pipe pipe, bool enable)
 332 {
 333         struct drm_i915_private *dev_priv = dev->dev_private;
 334         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 335         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 336         unsigned long flags;
 337         bool ret;
 338
 339         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 340
 341         ret = !intel_crtc->cpu_fifo_underrun_disabled;
 342
 343         if (enable == ret)
 344                 goto done;
 345
 346         intel_crtc->cpu_fifo_underrun_disabled = !enable;
 347
 348         if (IS_GEN5(dev) || IS_GEN6(dev))
 349                 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
 350         else if (IS_GEN7(dev))
 351                 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
 352
 353 done:
 354         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 355         return ret;
 356 }
 357
 358 /**
 359  * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
 360  * @dev: drm device
 361  * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
 362  * @enable: true if we want to report FIFO underrun errors, false otherwise
 363  *
 364  * This function makes us disable or enable PCH fifo underruns for a specific
 365  * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
 366  * underrun reporting for one transcoder may also disable all the other PCH
 367  * error interruts for the other transcoders, due to the fact that there's just
 368  * one interrupt mask/enable bit for all the transcoders.
 369  *
 370  * Returns the previous state of underrun reporting.
 371  */
 372 bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
 373                                            enum transcoder pch_transcoder,
 374                                            bool enable)
 375 {
 376         struct drm_i915_private *dev_priv = dev->dev_private;
 377         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
 378         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 379         unsigned long flags;
 380         bool ret;
 381
 382         /*
 383          * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT
 384          * has only one pch transcoder A that all pipes can use. To avoid racy
 385          * pch transcoder -> pipe lookups from interrupt code simply store the
 386          * underrun statistics in crtc A. Since we never expose this anywhere
 387          * nor use it outside of the fifo underrun code here using the "wrong"
 388          * crtc on LPT won't cause issues.
 389          */
 390
 391         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 392
 393         ret = !intel_crtc->pch_fifo_underrun_disabled;
 394
 395         if (enable == ret)
 396                 goto done;
 397
 398         intel_crtc->pch_fifo_underrun_disabled = !enable;
 399
 400         if (HAS_PCH_IBX(dev))
 401                 ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
 402         else
 403                 cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
 404
 405 done:
 406         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 407         return ret;
 408 }
 409
 410
 411 void
 412 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 413 {
 414         u32 reg = PIPESTAT(pipe);
 415         u32 pipestat = I915_READ(reg) & 0x7fff0000;
 416
 417         assert_spin_locked(&dev_priv->irq_lock);
 418
 419         if ((pipestat & mask) == mask)
 420                 return;
 421
 422         /* Enable the interrupt, clear any pending status */
 423         pipestat |= mask | (mask >> 16);
 424         I915_WRITE(reg, pipestat);
 425         POSTING_READ(reg);
 426 }
 427
 428 void
 429 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 430 {
 431         u32 reg = PIPESTAT(pipe);
 432         u32 pipestat = I915_READ(reg) & 0x7fff0000;
 433
 434         assert_spin_locked(&dev_priv->irq_lock);
 435
 436         if ((pipestat & mask) == 0)
 437                 return;
 438
 439         pipestat &= ~mask;
 440         I915_WRITE(reg, pipestat);
 441         POSTING_READ(reg);
 442 }
 443
 444 /**
 445  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 446  */
 447 static void i915_enable_asle_pipestat(struct drm_device *dev)
 448 {
 449         drm_i915_private_t *dev_priv = dev->dev_private;
 450         unsigned long irqflags;
 451
 452         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 453                 return;
 454
 455         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 456
 457         i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE);
 458         if (INTEL_INFO(dev)->gen >= 4)
 459                 i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE);
 460
 461         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 462 }
 463
 464 /**
 465  * i915_pipe_enabled - check if a pipe is enabled
 466  * @dev: DRM device
 467  * @pipe: pipe to check
 468  *
 469  * Reading certain registers when the pipe is disabled can hang the chip.
 470  * Use this routine to make sure the PLL is running and the pipe is active
 471  * before reading such registers if unsure.
 472  */
 473 static int
 474 i915_pipe_enabled(struct drm_device *dev, int pipe)
 475 {
 476         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 477
 478         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 479                 /* Locking is horribly broken here, but whatever. */
 480                 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 481                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 482
 483                 return intel_crtc->active;
 484         } else {
 485                 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
 486         }
 487 }
 488
 489 /* Called from drm generic code, passed a 'crtc', which
 490  * we use as a pipe index
 491  */
 492 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 493 {
 494         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 495         unsigned long high_frame;
 496         unsigned long low_frame;
 497         u32 high1, high2, low;
 498
 499         if (!i915_pipe_enabled(dev, pipe)) {
 500                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 501                                 "pipe %c\n", pipe_name(pipe));
 502                 return 0;
 503         }
 504
 505         high_frame = PIPEFRAME(pipe);
 506         low_frame = PIPEFRAMEPIXEL(pipe);
 507
 508         /*
 509          * High & low register fields aren't synchronized, so make sure
 510          * we get a low value that's stable across two reads of the high
 511          * register.
 512          */
 513         do {
 514                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 515                 low   = I915_READ(low_frame)  & PIPE_FRAME_LOW_MASK;
 516                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 517         } while (high1 != high2);
 518
 519         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 520         low >>= PIPE_FRAME_LOW_SHIFT;
 521         return (high1 << 8) | low;
 522 }
 523
 524 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 525 {
 526         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 527         int reg = PIPE_FRMCOUNT_GM45(pipe);
 528
 529         if (!i915_pipe_enabled(dev, pipe)) {
 530                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 531                                  "pipe %c\n", pipe_name(pipe));
 532                 return 0;
 533         }
 534
 535         return I915_READ(reg);
 536 }
 537
 538 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 539                              int *vpos, int *hpos)
 540 {
 541         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 542         u32 vbl = 0, position = 0;
 543         int vbl_start, vbl_end, htotal, vtotal;
 544         bool in_vbl = true;
 545         int ret = 0;
 546         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
 547                                                                       pipe);
 548
 549         if (!i915_pipe_enabled(dev, pipe)) {
 550                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 551                                  "pipe %c\n", pipe_name(pipe));
 552                 return 0;
 553         }
 554
 555         /* Get vtotal. */
 556         vtotal = 1 + ((I915_READ(VTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 557
 558         if (INTEL_INFO(dev)->gen >= 4) {
 559                 /* No obvious pixelcount register. Only query vertical
 560                  * scanout position from Display scan line register.
 561                  */
 562                 position = I915_READ(PIPEDSL(pipe));
 563
 564                 /* Decode into vertical scanout position. Don't have
 565                  * horizontal scanout position.
 566                  */
 567                 *vpos = position & 0x1fff;
 568                 *hpos = 0;
 569         } else {
 570                 /* Have access to pixelcount since start of frame.
 571                  * We can split this into vertical and horizontal
 572                  * scanout position.
 573                  */
 574                 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 575
 576                 htotal = 1 + ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 577                 *vpos = position / htotal;
 578                 *hpos = position - (*vpos * htotal);
 579         }
 580
 581         /* Query vblank area. */
 582         vbl = I915_READ(VBLANK(cpu_transcoder));
 583
 584         /* Test position against vblank region. */
 585         vbl_start = vbl & 0x1fff;
 586         vbl_end = (vbl >> 16) & 0x1fff;
 587
 588         if ((*vpos < vbl_start) || (*vpos > vbl_end))
 589                 in_vbl = false;
 590
 591         /* Inside "upper part" of vblank area? Apply corrective offset: */
 592         if (in_vbl && (*vpos >= vbl_start))
 593                 *vpos = *vpos - vtotal;
 594
 595         /* Readouts valid? */
 596         if (vbl > 0)
 597                 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 598
 599         /* In vblank? */
 600         if (in_vbl)
 601                 ret |= DRM_SCANOUTPOS_INVBL;
 602
 603         return ret;
 604 }
 605
 606 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 607                               int *max_error,
 608                               struct timeval *vblank_time,
 609                               unsigned flags)
 610 {
 611         struct drm_crtc *crtc;
 612
 613         if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 614                 DRM_ERROR("Invalid crtc %d\n", pipe);
 615                 return -EINVAL;
 616         }
 617
 618         /* Get drm_crtc to timestamp: */
 619         crtc = intel_get_crtc_for_pipe(dev, pipe);
 620         if (crtc == NULL) {
 621                 DRM_ERROR("Invalid crtc %d\n", pipe);
 622                 return -EINVAL;
 623         }
 624
 625         if (!crtc->enabled) {
 626                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 627                 return -EBUSY;
 628         }
 629
 630         /* Helper routine in DRM core does all the work: */
 631         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 632                                                      vblank_time, flags,
 633                                                      crtc);
 634 }
 635
 636 static int intel_hpd_irq_event(struct drm_device *dev, struct drm_connector *connector)
 637 {
 638         enum drm_connector_status old_status;
 639
 640         WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
 641         old_status = connector->status;
 642
 643         connector->status = connector->funcs->detect(connector, false);
 644         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
 645                       connector->base.id,
 646                       drm_get_connector_name(connector),
 647                       old_status, connector->status);
 648         return (old_status != connector->status);
 649 }
 650
 651 /*
 652  * Handle hotplug events outside the interrupt handler proper.
 653  */
 654 #define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
 655
 656 static void i915_hotplug_work_func(struct work_struct *work)
 657 {
 658         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 659                                                     hotplug_work);
 660         struct drm_device *dev = dev_priv->dev;
 661         struct drm_mode_config *mode_config = &dev->mode_config;
 662         struct intel_connector *intel_connector;
 663         struct intel_encoder *intel_encoder;
 664         struct drm_connector *connector;
 665         unsigned long irqflags;
 666         bool hpd_disabled = false;
 667         bool changed = false;
 668         u32 hpd_event_bits;
 669
 670         /* HPD irq before everything is fully set up. */
 671         if (!dev_priv->enable_hotplug_processing)
 672                 return;
 673
 674         mutex_lock(&mode_config->mutex);
 675         DRM_DEBUG_KMS("running encoder hotplug functions\n");
 676
 677         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 678
 679         hpd_event_bits = dev_priv->hpd_event_bits;
 680         dev_priv->hpd_event_bits = 0;
 681         list_for_each_entry(connector, &mode_config->connector_list, head) {
 682                 intel_connector = to_intel_connector(connector);
 683                 intel_encoder = intel_connector->encoder;
 684                 if (intel_encoder->hpd_pin > HPD_NONE &&
 685                     dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
 686                     connector->polled == DRM_CONNECTOR_POLL_HPD) {
 687                         DRM_INFO("HPD interrupt storm detected on connector %s: "
 688                                  "switching from hotplug detection to polling\n",
 689                                 drm_get_connector_name(connector));
 690                         dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
 691                         connector->polled = DRM_CONNECTOR_POLL_CONNECT
 692                                 | DRM_CONNECTOR_POLL_DISCONNECT;
 693                         hpd_disabled = true;
 694                 }
 695                 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
 696                         DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
 697                                       drm_get_connector_name(connector), intel_encoder->hpd_pin);
 698                 }
 699         }
 700          /* if there were no outputs to poll, poll was disabled,
 701           * therefore make sure it's enabled when disabling HPD on
 702           * some connectors */
 703         if (hpd_disabled) {
 704                 drm_kms_helper_poll_enable(dev);
 705                 mod_timer(&dev_priv->hotplug_reenable_timer,
 706                           jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
 707         }
 708
 709         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 710
 711         list_for_each_entry(connector, &mode_config->connector_list, head) {
 712                 intel_connector = to_intel_connector(connector);
 713                 intel_encoder = intel_connector->encoder;
 714                 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
 715                         if (intel_encoder->hot_plug)
 716                                 intel_encoder->hot_plug(intel_encoder);
 717                         if (intel_hpd_irq_event(dev, connector))
 718                                 changed = true;
 719                 }
 720         }
 721         mutex_unlock(&mode_config->mutex);
 722
 723         if (changed)
 724                 drm_kms_helper_hotplug_event(dev);
 725 }
 726
 727 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 728 {
 729         drm_i915_private_t *dev_priv = dev->dev_private;
 730         u32 busy_up, busy_down, max_avg, min_avg;
 731         u8 new_delay;
 732
 733         spin_lock(&mchdev_lock);
 734
 735         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 736
 737         new_delay = dev_priv->ips.cur_delay;
 738
 739         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 740         busy_up = I915_READ(RCPREVBSYTUPAVG);
 741         busy_down = I915_READ(RCPREVBSYTDNAVG);
 742         max_avg = I915_READ(RCBMAXAVG);
 743         min_avg = I915_READ(RCBMINAVG);
 744
 745         /* Handle RCS change request from hw */
 746         if (busy_up > max_avg) {
 747                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 748                         new_delay = dev_priv->ips.cur_delay - 1;
 749                 if (new_delay < dev_priv->ips.max_delay)
 750                         new_delay = dev_priv->ips.max_delay;
 751         } else if (busy_down < min_avg) {
 752                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 753                         new_delay = dev_priv->ips.cur_delay + 1;
 754                 if (new_delay > dev_priv->ips.min_delay)
 755                         new_delay = dev_priv->ips.min_delay;
 756         }
 757
 758         if (ironlake_set_drps(dev, new_delay))
 759                 dev_priv->ips.cur_delay = new_delay;
 760
 761         spin_unlock(&mchdev_lock);
 762
 763         return;
 764 }
 765
 766 static void notify_ring(struct drm_device *dev,
 767                         struct intel_ring_buffer *ring)
 768 {
 769         if (ring->obj == NULL)
 770                 return;
 771
 772         trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
 773
 774         wake_up_all(&ring->irq_queue);
 775         i915_queue_hangcheck(dev);
 776 }
 777
 778 static void gen6_pm_rps_work(struct work_struct *work)
 779 {
 780         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 781                                                     rps.work);
 782         u32 pm_iir;
 783         u8 new_delay;
 784
 785         spin_lock_irq(&dev_priv->irq_lock);
 786         pm_iir = dev_priv->rps.pm_iir;
 787         dev_priv->rps.pm_iir = 0;
 788         /* Make sure not to corrupt PMIMR state used by ringbuffer code */
 789         snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
 790         spin_unlock_irq(&dev_priv->irq_lock);
 791
 792         if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0)
 793                 return;
 794
 795         mutex_lock(&dev_priv->rps.hw_lock);
 796
 797         if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
 798                 new_delay = dev_priv->rps.cur_delay + 1;
 799
 800                 /*
 801                  * For better performance, jump directly
 802                  * to RPe if we're below it.
 803                  */
 804                 if (IS_VALLEYVIEW(dev_priv->dev) &&
 805                     dev_priv->rps.cur_delay < dev_priv->rps.rpe_delay)
 806                         new_delay = dev_priv->rps.rpe_delay;
 807         } else
 808                 new_delay = dev_priv->rps.cur_delay - 1;
 809
 810         /* sysfs frequency interfaces may have snuck in while servicing the
 811          * interrupt
 812          */
 813         if (new_delay >= dev_priv->rps.min_delay &&
 814             new_delay <= dev_priv->rps.max_delay) {
 815                 if (IS_VALLEYVIEW(dev_priv->dev))
 816                         valleyview_set_rps(dev_priv->dev, new_delay);
 817                 else
 818                         gen6_set_rps(dev_priv->dev, new_delay);
 819         }
 820
 821         if (IS_VALLEYVIEW(dev_priv->dev)) {
 822                 /*
 823                  * On VLV, when we enter RC6 we may not be at the minimum
 824                  * voltage level, so arm a timer to check.  It should only
 825                  * fire when there's activity or once after we've entered
 826                  * RC6, and then won't be re-armed until the next RPS interrupt.
 827                  */
 828                 mod_delayed_work(dev_priv->wq, &dev_priv->rps.vlv_work,
 829                                  msecs_to_jiffies(100));
 830         }
 831
 832         mutex_unlock(&dev_priv->rps.hw_lock);
 833 }
 834
 835
 836 /**
 837  * ivybridge_parity_work - Workqueue called when a parity error interrupt
 838  * occurred.
 839  * @work: workqueue struct
 840  *
 841  * Doesn't actually do anything except notify userspace. As a consequence of
 842  * this event, userspace should try to remap the bad rows since statistically
 843  * it is likely the same row is more likely to go bad again.
 844  */
 845 static void ivybridge_parity_work(struct work_struct *work)
 846 {
 847         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 848                                                     l3_parity.error_work);
 849         u32 error_status, row, bank, subbank;
 850         char *parity_event[5];
 851         uint32_t misccpctl;
 852         unsigned long flags;
 853
 854         /* We must turn off DOP level clock gating to access the L3 registers.
 855          * In order to prevent a get/put style interface, acquire struct mutex
 856          * any time we access those registers.
 857          */
 858         mutex_lock(&dev_priv->dev->struct_mutex);
 859
 860         misccpctl = I915_READ(GEN7_MISCCPCTL);
 861         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 862         POSTING_READ(GEN7_MISCCPCTL);
 863
 864         error_status = I915_READ(GEN7_L3CDERRST1);
 865         row = GEN7_PARITY_ERROR_ROW(error_status);
 866         bank = GEN7_PARITY_ERROR_BANK(error_status);
 867         subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
 868
 869         I915_WRITE(GEN7_L3CDERRST1, GEN7_PARITY_ERROR_VALID |
 870                                     GEN7_L3CDERRST1_ENABLE);
 871         POSTING_READ(GEN7_L3CDERRST1);
 872
 873         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 874
 875         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 876         ilk_enable_gt_irq(dev_priv, GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
 877         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 878
 879         mutex_unlock(&dev_priv->dev->struct_mutex);
 880
 881         parity_event[0] = I915_L3_PARITY_UEVENT "=1";
 882         parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
 883         parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
 884         parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
 885         parity_event[4] = NULL;
 886
 887         kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
 888                            KOBJ_CHANGE, parity_event);
 889
 890         DRM_DEBUG("Parity error: Row = %d, Bank = %d, Sub bank = %d.\n",
 891                   row, bank, subbank);
 892
 893         kfree(parity_event[3]);
 894         kfree(parity_event[2]);
 895         kfree(parity_event[1]);
 896 }
 897
 898 static void ivybridge_parity_error_irq_handler(struct drm_device *dev)
 899 {
 900         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 901
 902         if (!HAS_L3_GPU_CACHE(dev))
 903                 return;
 904
 905         spin_lock(&dev_priv->irq_lock);
 906         ilk_disable_gt_irq(dev_priv, GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
 907         spin_unlock(&dev_priv->irq_lock);
 908
 909         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
 910 }
 911
 912 static void ilk_gt_irq_handler(struct drm_device *dev,
 913                                struct drm_i915_private *dev_priv,
 914                                u32 gt_iir)
 915 {
 916         if (gt_iir &
 917             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
 918                 notify_ring(dev, &dev_priv->ring[RCS]);
 919         if (gt_iir & ILK_BSD_USER_INTERRUPT)
 920                 notify_ring(dev, &dev_priv->ring[VCS]);
 921 }
 922
 923 static void snb_gt_irq_handler(struct drm_device *dev,
 924                                struct drm_i915_private *dev_priv,
 925                                u32 gt_iir)
 926 {
 927
 928         if (gt_iir &
 929             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
 930                 notify_ring(dev, &dev_priv->ring[RCS]);
 931         if (gt_iir & GT_BSD_USER_INTERRUPT)
 932                 notify_ring(dev, &dev_priv->ring[VCS]);
 933         if (gt_iir & GT_BLT_USER_INTERRUPT)
 934                 notify_ring(dev, &dev_priv->ring[BCS]);
 935
 936         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 937                       GT_BSD_CS_ERROR_INTERRUPT |
 938                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
 939                 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
 940                 i915_handle_error(dev, false);
 941         }
 942
 943         if (gt_iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
 944                 ivybridge_parity_error_irq_handler(dev);
 945 }
 946
 947 /* Legacy way of handling PM interrupts */
 948 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv,
 949                                  u32 pm_iir)
 950 {
 951         /*
 952          * IIR bits should never already be set because IMR should
 953          * prevent an interrupt from being shown in IIR. The warning
 954          * displays a case where we've unsafely cleared
 955          * dev_priv->rps.pm_iir. Although missing an interrupt of the same
 956          * type is not a problem, it displays a problem in the logic.
 957          *
 958          * The mask bit in IMR is cleared by dev_priv->rps.work.
 959          */
 960
 961         spin_lock(&dev_priv->irq_lock);
 962         dev_priv->rps.pm_iir |= pm_iir;
 963         snb_set_pm_irq(dev_priv, dev_priv->rps.pm_iir);
 964         spin_unlock(&dev_priv->irq_lock);
 965
 966         queue_work(dev_priv->wq, &dev_priv->rps.work);
 967 }
 968
 969 #define HPD_STORM_DETECT_PERIOD 1000
 970 #define HPD_STORM_THRESHOLD 5
 971
 972 static inline void intel_hpd_irq_handler(struct drm_device *dev,
 973                                          u32 hotplug_trigger,
 974                                          const u32 *hpd)
 975 {
 976         drm_i915_private_t *dev_priv = dev->dev_private;
 977         int i;
 978         bool storm_detected = false;
 979
 980         if (!hotplug_trigger)
 981                 return;
 982
 983         spin_lock(&dev_priv->irq_lock);
 984         for (i = 1; i < HPD_NUM_PINS; i++) {
 985
 986                 WARN(((hpd[i] & hotplug_trigger) &&
 987                       dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED),
 988                      "Received HPD interrupt although disabled\n");
 989
 990                 if (!(hpd[i] & hotplug_trigger) ||
 991                     dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
 992                         continue;
 993
 994                 dev_priv->hpd_event_bits |= (1 << i);
 995                 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
 996                                    dev_priv->hpd_stats[i].hpd_last_jiffies
 997                                    + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
 998                         dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
 999                         dev_priv->hpd_stats[i].hpd_cnt = 0;
1000                         DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
1001                 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
1002                         dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
1003                         dev_priv->hpd_event_bits &= ~(1 << i);
1004                         DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
1005                         storm_detected = true;
1006                 } else {
1007                         dev_priv->hpd_stats[i].hpd_cnt++;
1008                         DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
1009                                       dev_priv->hpd_stats[i].hpd_cnt);
1010                 }
1011         }
1012
1013         if (storm_detected)
1014                 dev_priv->display.hpd_irq_setup(dev);
1015         spin_unlock(&dev_priv->irq_lock);
1016
1017         queue_work(dev_priv->wq,
1018                    &dev_priv->hotplug_work);
1019 }
1020
1021 static void gmbus_irq_handler(struct drm_device *dev)
1022 {
1023         struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1024
1025         wake_up_all(&dev_priv->gmbus_wait_queue);
1026 }
1027
1028 static void dp_aux_irq_handler(struct drm_device *dev)
1029 {
1030         struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1031
1032         wake_up_all(&dev_priv->gmbus_wait_queue);
1033 }
1034
1035 /* Unlike gen6_rps_irq_handler() from which this function is originally derived,
1036  * we must be able to deal with other PM interrupts. This is complicated because
1037  * of the way in which we use the masks to defer the RPS work (which for
1038  * posterity is necessary because of forcewake).
1039  */
1040 static void hsw_pm_irq_handler(struct drm_i915_private *dev_priv,
1041                                u32 pm_iir)
1042 {
1043         if (pm_iir & GEN6_PM_RPS_EVENTS) {
1044                 spin_lock(&dev_priv->irq_lock);
1045                 dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
1046                 snb_set_pm_irq(dev_priv, dev_priv->rps.pm_iir);
1047                 /* never want to mask useful interrupts. */
1048                 WARN_ON(I915_READ_NOTRACE(GEN6_PMIMR) & ~GEN6_PM_RPS_EVENTS);
1049                 spin_unlock(&dev_priv->irq_lock);
1050
1051                 queue_work(dev_priv->wq, &dev_priv->rps.work);
1052         }
1053
1054         if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1055                 notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
1056
1057         if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
1058                 DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir);
1059                 i915_handle_error(dev_priv->dev, false);
1060         }
1061 }
1062
1063 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1064 {
1065         struct drm_device *dev = (struct drm_device *) arg;
1066         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1067         u32 iir, gt_iir, pm_iir;
1068         irqreturn_t ret = IRQ_NONE;
1069         unsigned long irqflags;
1070         int pipe;
1071         u32 pipe_stats[I915_MAX_PIPES];
1072
1073         atomic_inc(&dev_priv->irq_received);
1074
1075         while (true) {
1076                 iir = I915_READ(VLV_IIR);
1077                 gt_iir = I915_READ(GTIIR);
1078                 pm_iir = I915_READ(GEN6_PMIIR);
1079
1080                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1081                         goto out;
1082
1083                 ret = IRQ_HANDLED;
1084
1085                 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1086
1087                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1088                 for_each_pipe(pipe) {
1089                         int reg = PIPESTAT(pipe);
1090                         pipe_stats[pipe] = I915_READ(reg);
1091
1092                         /*
1093                          * Clear the PIPE*STAT regs before the IIR
1094                          */
1095                         if (pipe_stats[pipe] & 0x8000ffff) {
1096                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1097                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
1098                                                          pipe_name(pipe));
1099                                 I915_WRITE(reg, pipe_stats[pipe]);
1100                         }
1101                 }
1102                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1103
1104                 for_each_pipe(pipe) {
1105                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1106                                 drm_handle_vblank(dev, pipe);
1107
1108                         if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
1109                                 intel_prepare_page_flip(dev, pipe);
1110                                 intel_finish_page_flip(dev, pipe);
1111                         }
1112                 }
1113
1114                 /* Consume port.  Then clear IIR or we'll miss events */
1115                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
1116                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1117                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1118
1119                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1120                                          hotplug_status);
1121
1122                         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
1123
1124                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1125                         I915_READ(PORT_HOTPLUG_STAT);
1126                 }
1127
1128                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1129                         gmbus_irq_handler(dev);
1130
1131                 if (pm_iir & GEN6_PM_RPS_EVENTS)
1132                         gen6_rps_irq_handler(dev_priv, pm_iir);
1133
1134                 I915_WRITE(GTIIR, gt_iir);
1135                 I915_WRITE(GEN6_PMIIR, pm_iir);
1136                 I915_WRITE(VLV_IIR, iir);
1137         }
1138
1139 out:
1140         return ret;
1141 }
1142
1143 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1144 {
1145         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1146         int pipe;
1147         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1148
1149         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1150
1151         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1152                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1153                                SDE_AUDIO_POWER_SHIFT);
1154                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1155                                  port_name(port));
1156         }
1157
1158         if (pch_iir & SDE_AUX_MASK)
1159                 dp_aux_irq_handler(dev);
1160
1161         if (pch_iir & SDE_GMBUS)
1162                 gmbus_irq_handler(dev);
1163
1164         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1165                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1166
1167         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1168                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1169
1170         if (pch_iir & SDE_POISON)
1171                 DRM_ERROR("PCH poison interrupt\n");
1172
1173         if (pch_iir & SDE_FDI_MASK)
1174                 for_each_pipe(pipe)
1175                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1176                                          pipe_name(pipe),
1177                                          I915_READ(FDI_RX_IIR(pipe)));
1178
1179         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1180                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1181
1182         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1183                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1184
1185         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1186                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1187                                                           false))
1188                         DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1189
1190         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1191                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1192                                                           false))
1193                         DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1194 }
1195
1196 static void ivb_err_int_handler(struct drm_device *dev)
1197 {
1198         struct drm_i915_private *dev_priv = dev->dev_private;
1199         u32 err_int = I915_READ(GEN7_ERR_INT);
1200
1201         if (err_int & ERR_INT_POISON)
1202                 DRM_ERROR("Poison interrupt\n");
1203
1204         if (err_int & ERR_INT_FIFO_UNDERRUN_A)
1205                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1206                         DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1207
1208         if (err_int & ERR_INT_FIFO_UNDERRUN_B)
1209                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1210                         DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1211
1212         if (err_int & ERR_INT_FIFO_UNDERRUN_C)
1213                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_C, false))
1214                         DRM_DEBUG_DRIVER("Pipe C FIFO underrun\n");
1215
1216         I915_WRITE(GEN7_ERR_INT, err_int);
1217 }
1218
1219 static void cpt_serr_int_handler(struct drm_device *dev)
1220 {
1221         struct drm_i915_private *dev_priv = dev->dev_private;
1222         u32 serr_int = I915_READ(SERR_INT);
1223
1224         if (serr_int & SERR_INT_POISON)
1225                 DRM_ERROR("PCH poison interrupt\n");
1226
1227         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1228                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1229                                                           false))
1230                         DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1231
1232         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1233                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1234                                                           false))
1235                         DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1236
1237         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1238                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
1239                                                           false))
1240                         DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
1241
1242         I915_WRITE(SERR_INT, serr_int);
1243 }
1244
1245 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1246 {
1247         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1248         int pipe;
1249         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1250
1251         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1252
1253         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1254                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1255                                SDE_AUDIO_POWER_SHIFT_CPT);
1256                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1257                                  port_name(port));
1258         }
1259
1260         if (pch_iir & SDE_AUX_MASK_CPT)
1261                 dp_aux_irq_handler(dev);
1262
1263         if (pch_iir & SDE_GMBUS_CPT)
1264                 gmbus_irq_handler(dev);
1265
1266         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1267                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1268
1269         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1270                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1271
1272         if (pch_iir & SDE_FDI_MASK_CPT)
1273                 for_each_pipe(pipe)
1274                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1275                                          pipe_name(pipe),
1276                                          I915_READ(FDI_RX_IIR(pipe)));
1277
1278         if (pch_iir & SDE_ERROR_CPT)
1279                 cpt_serr_int_handler(dev);
1280 }
1281
1282 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1283 {
1284         struct drm_i915_private *dev_priv = dev->dev_private;
1285
1286         if (de_iir & DE_AUX_CHANNEL_A)
1287                 dp_aux_irq_handler(dev);
1288
1289         if (de_iir & DE_GSE)
1290                 intel_opregion_asle_intr(dev);
1291
1292         if (de_iir & DE_PIPEA_VBLANK)
1293                 drm_handle_vblank(dev, 0);
1294
1295         if (de_iir & DE_PIPEB_VBLANK)
1296                 drm_handle_vblank(dev, 1);
1297
1298         if (de_iir & DE_POISON)
1299                 DRM_ERROR("Poison interrupt\n");
1300
1301         if (de_iir & DE_PIPEA_FIFO_UNDERRUN)
1302                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1303                         DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1304
1305         if (de_iir & DE_PIPEB_FIFO_UNDERRUN)
1306                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1307                         DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1308
1309         if (de_iir & DE_PLANEA_FLIP_DONE) {
1310                 intel_prepare_page_flip(dev, 0);
1311                 intel_finish_page_flip_plane(dev, 0);
1312         }
1313
1314         if (de_iir & DE_PLANEB_FLIP_DONE) {
1315                 intel_prepare_page_flip(dev, 1);
1316                 intel_finish_page_flip_plane(dev, 1);
1317         }
1318
1319         /* check event from PCH */
1320         if (de_iir & DE_PCH_EVENT) {
1321                 u32 pch_iir = I915_READ(SDEIIR);
1322
1323                 if (HAS_PCH_CPT(dev))
1324                         cpt_irq_handler(dev, pch_iir);
1325                 else
1326                         ibx_irq_handler(dev, pch_iir);
1327
1328                 /* should clear PCH hotplug event before clear CPU irq */
1329                 I915_WRITE(SDEIIR, pch_iir);
1330         }
1331
1332         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1333                 ironlake_rps_change_irq_handler(dev);
1334 }
1335
1336 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1337 {
1338         struct drm_i915_private *dev_priv = dev->dev_private;
1339         int i;
1340
1341         if (de_iir & DE_ERR_INT_IVB)
1342                 ivb_err_int_handler(dev);
1343
1344         if (de_iir & DE_AUX_CHANNEL_A_IVB)
1345                 dp_aux_irq_handler(dev);
1346
1347         if (de_iir & DE_GSE_IVB)
1348                 intel_opregion_asle_intr(dev);
1349
1350         for (i = 0; i < 3; i++) {
1351                 if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
1352                         drm_handle_vblank(dev, i);
1353                 if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
1354                         intel_prepare_page_flip(dev, i);
1355                         intel_finish_page_flip_plane(dev, i);
1356                 }
1357         }
1358
1359         /* check event from PCH */
1360         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1361                 u32 pch_iir = I915_READ(SDEIIR);
1362
1363                 cpt_irq_handler(dev, pch_iir);
1364
1365                 /* clear PCH hotplug event before clear CPU irq */
1366                 I915_WRITE(SDEIIR, pch_iir);
1367         }
1368 }
1369
1370 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
1371 {
1372         struct drm_device *dev = (struct drm_device *) arg;
1373         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1374         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
1375         irqreturn_t ret = IRQ_NONE;
1376         bool err_int_reenable = false;
1377
1378         atomic_inc(&dev_priv->irq_received);
1379
1380         /* We get interrupts on unclaimed registers, so check for this before we
1381          * do any I915_{READ,WRITE}. */
1382         intel_uncore_check_errors(dev);
1383
1384         /* disable master interrupt before clearing iir  */
1385         de_ier = I915_READ(DEIER);
1386         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1387         POSTING_READ(DEIER);
1388
1389         /* Disable south interrupts. We'll only write to SDEIIR once, so further
1390          * interrupts will will be stored on its back queue, and then we'll be
1391          * able to process them after we restore SDEIER (as soon as we restore
1392          * it, we'll get an interrupt if SDEIIR still has something to process
1393          * due to its back queue). */
1394         if (!HAS_PCH_NOP(dev)) {
1395                 sde_ier = I915_READ(SDEIER);
1396                 I915_WRITE(SDEIER, 0);
1397                 POSTING_READ(SDEIER);
1398         }
1399
1400         /* On Haswell, also mask ERR_INT because we don't want to risk
1401          * generating "unclaimed register" interrupts from inside the interrupt
1402          * handler. */
1403         if (IS_HASWELL(dev)) {
1404                 spin_lock(&dev_priv->irq_lock);
1405                 err_int_reenable = ~dev_priv->irq_mask & DE_ERR_INT_IVB;
1406                 if (err_int_reenable)
1407                         ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
1408                 spin_unlock(&dev_priv->irq_lock);
1409         }
1410
1411         gt_iir = I915_READ(GTIIR);
1412         if (gt_iir) {
1413                 if (INTEL_INFO(dev)->gen >= 6)
1414                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1415                 else
1416                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1417                 I915_WRITE(GTIIR, gt_iir);
1418                 ret = IRQ_HANDLED;
1419         }
1420
1421         de_iir = I915_READ(DEIIR);
1422         if (de_iir) {
1423                 if (INTEL_INFO(dev)->gen >= 7)
1424                         ivb_display_irq_handler(dev, de_iir);
1425                 else
1426                         ilk_display_irq_handler(dev, de_iir);
1427                 I915_WRITE(DEIIR, de_iir);
1428                 ret = IRQ_HANDLED;
1429         }
1430
1431         if (INTEL_INFO(dev)->gen >= 6) {
1432                 u32 pm_iir = I915_READ(GEN6_PMIIR);
1433                 if (pm_iir) {
1434                         if (IS_HASWELL(dev))
1435                                 hsw_pm_irq_handler(dev_priv, pm_iir);
1436                         else if (pm_iir & GEN6_PM_RPS_EVENTS)
1437                                 gen6_rps_irq_handler(dev_priv, pm_iir);
1438                         I915_WRITE(GEN6_PMIIR, pm_iir);
1439                         ret = IRQ_HANDLED;
1440                 }
1441         }
1442
1443         if (err_int_reenable) {
1444                 spin_lock(&dev_priv->irq_lock);
1445                 if (ivb_can_enable_err_int(dev))
1446                         ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
1447                 spin_unlock(&dev_priv->irq_lock);
1448         }
1449
1450         I915_WRITE(DEIER, de_ier);
1451         POSTING_READ(DEIER);
1452         if (!HAS_PCH_NOP(dev)) {
1453                 I915_WRITE(SDEIER, sde_ier);
1454                 POSTING_READ(SDEIER);
1455         }
1456
1457         return ret;
1458 }
1459
1460 /**
1461  * i915_error_work_func - do process context error handling work
1462  * @work: work struct
1463  *
1464  * Fire an error uevent so userspace can see that a hang or error
1465  * was detected.
1466  */
1467 static void i915_error_work_func(struct work_struct *work)
1468 {
1469         struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
1470                                                     work);
1471         drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
1472                                                     gpu_error);
1473         struct drm_device *dev = dev_priv->dev;
1474         struct intel_ring_buffer *ring;
1475         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
1476         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
1477         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
1478         int i, ret;
1479
1480         kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
1481
1482         /*
1483          * Note that there's only one work item which does gpu resets, so we
1484          * need not worry about concurrent gpu resets potentially incrementing
1485          * error->reset_counter twice. We only need to take care of another
1486          * racing irq/hangcheck declaring the gpu dead for a second time. A
1487          * quick check for that is good enough: schedule_work ensures the
1488          * correct ordering between hang detection and this work item, and since
1489          * the reset in-progress bit is only ever set by code outside of this
1490          * work we don't need to worry about any other races.
1491          */
1492         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
1493                 DRM_DEBUG_DRIVER("resetting chip\n");
1494                 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
1495                                    reset_event);
1496
1497                 ret = i915_reset(dev);
1498
1499                 if (ret == 0) {
1500                         /*
1501                          * After all the gem state is reset, increment the reset
1502                          * counter and wake up everyone waiting for the reset to
1503                          * complete.
1504                          *
1505                          * Since unlock operations are a one-sided barrier only,
1506                          * we need to insert a barrier here to order any seqno
1507                          * updates before
1508                          * the counter increment.
1509                          */
1510                         smp_mb__before_atomic_inc();
1511                         atomic_inc(&dev_priv->gpu_error.reset_counter);
1512
1513                         kobject_uevent_env(&dev->primary->kdev.kobj,
1514                                            KOBJ_CHANGE, reset_done_event);
1515                 } else {
1516                         atomic_set(&error->reset_counter, I915_WEDGED);
1517                 }
1518
1519                 for_each_ring(ring, dev_priv, i)
1520                         wake_up_all(&ring->irq_queue);
1521
1522                 intel_display_handle_reset(dev);
1523
1524                 wake_up_all(&dev_priv->gpu_error.reset_queue);
1525         }
1526 }
1527
1528 static void i915_report_and_clear_eir(struct drm_device *dev)
1529 {
1530         struct drm_i915_private *dev_priv = dev->dev_private;
1531         uint32_t instdone[I915_NUM_INSTDONE_REG];
1532         u32 eir = I915_READ(EIR);
1533         int pipe, i;
1534
1535         if (!eir)
1536                 return;
1537
1538         pr_err("render error detected, EIR: 0x%08x\n", eir);
1539
1540         i915_get_extra_instdone(dev, instdone);
1541
1542         if (IS_G4X(dev)) {
1543                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
1544                         u32 ipeir = I915_READ(IPEIR_I965);
1545
1546                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1547                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1548                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
1549                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1550                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1551                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1552                         I915_WRITE(IPEIR_I965, ipeir);
1553                         POSTING_READ(IPEIR_I965);
1554                 }
1555                 if (eir & GM45_ERROR_PAGE_TABLE) {
1556                         u32 pgtbl_err = I915_READ(PGTBL_ER);
1557                         pr_err("page table error\n");
1558                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1559                         I915_WRITE(PGTBL_ER, pgtbl_err);
1560                         POSTING_READ(PGTBL_ER);
1561                 }
1562         }
1563
1564         if (!IS_GEN2(dev)) {
1565                 if (eir & I915_ERROR_PAGE_TABLE) {
1566                         u32 pgtbl_err = I915_READ(PGTBL_ER);
1567                         pr_err("page table error\n");
1568                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1569                         I915_WRITE(PGTBL_ER, pgtbl_err);
1570                         POSTING_READ(PGTBL_ER);
1571                 }
1572         }
1573
1574         if (eir & I915_ERROR_MEMORY_REFRESH) {
1575                 pr_err("memory refresh error:\n");
1576                 for_each_pipe(pipe)
1577                         pr_err("pipe %c stat: 0x%08x\n",
1578                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
1579                 /* pipestat has already been acked */
1580         }
1581         if (eir & I915_ERROR_INSTRUCTION) {
1582                 pr_err("instruction error\n");
1583                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
1584                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
1585                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1586                 if (INTEL_INFO(dev)->gen < 4) {
1587                         u32 ipeir = I915_READ(IPEIR);
1588
1589                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
1590                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
1591                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
1592                         I915_WRITE(IPEIR, ipeir);
1593                         POSTING_READ(IPEIR);
1594                 } else {
1595                         u32 ipeir = I915_READ(IPEIR_I965);
1596
1597                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1598                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1599                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1600                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1601                         I915_WRITE(IPEIR_I965, ipeir);
1602                         POSTING_READ(IPEIR_I965);
1603                 }
1604         }
1605
1606         I915_WRITE(EIR, eir);
1607         POSTING_READ(EIR);
1608         eir = I915_READ(EIR);
1609         if (eir) {
1610                 /*
1611                  * some errors might have become stuck,
1612                  * mask them.
1613                  */
1614                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
1615                 I915_WRITE(EMR, I915_READ(EMR) | eir);
1616                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
1617         }
1618 }
1619
1620 /**
1621  * i915_handle_error - handle an error interrupt
1622  * @dev: drm device
1623  *
1624  * Do some basic checking of regsiter state at error interrupt time and
1625  * dump it to the syslog.  Also call i915_capture_error_state() to make
1626  * sure we get a record and make it available in debugfs.  Fire a uevent
1627  * so userspace knows something bad happened (should trigger collection
1628  * of a ring dump etc.).
1629  */
1630 void i915_handle_error(struct drm_device *dev, bool wedged)
1631 {
1632         struct drm_i915_private *dev_priv = dev->dev_private;
1633         struct intel_ring_buffer *ring;
1634         int i;
1635
1636         i915_capture_error_state(dev);
1637         i915_report_and_clear_eir(dev);
1638
1639         if (wedged) {
1640                 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
1641                                 &dev_priv->gpu_error.reset_counter);
1642
1643                 /*
1644                  * Wakeup waiting processes so that the reset work item
1645                  * doesn't deadlock trying to grab various locks.
1646                  */
1647                 for_each_ring(ring, dev_priv, i)
1648                         wake_up_all(&ring->irq_queue);
1649         }
1650
1651         queue_work(dev_priv->wq, &dev_priv->gpu_error.work);
1652 }
1653
1654 static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
1655 {
1656         drm_i915_private_t *dev_priv = dev->dev_private;
1657         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1658         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1659         struct drm_i915_gem_object *obj;
1660         struct intel_unpin_work *work;
1661         unsigned long flags;
1662         bool stall_detected;
1663
1664         /* Ignore early vblank irqs */
1665         if (intel_crtc == NULL)
1666                 return;
1667
1668         spin_lock_irqsave(&dev->event_lock, flags);
1669         work = intel_crtc->unpin_work;
1670
1671         if (work == NULL ||
1672             atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
1673             !work->enable_stall_check) {
1674                 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
1675                 spin_unlock_irqrestore(&dev->event_lock, flags);
1676                 return;
1677         }
1678
1679         /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
1680         obj = work->pending_flip_obj;
1681         if (INTEL_INFO(dev)->gen >= 4) {
1682                 int dspsurf = DSPSURF(intel_crtc->plane);
1683                 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
1684                                         i915_gem_obj_ggtt_offset(obj);
1685         } else {
1686                 int dspaddr = DSPADDR(intel_crtc->plane);
1687                 stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
1688                                                         crtc->y * crtc->fb->pitches[0] +
1689                                                         crtc->x * crtc->fb->bits_per_pixel/8);
1690         }
1691
1692         spin_unlock_irqrestore(&dev->event_lock, flags);
1693
1694         if (stall_detected) {
1695                 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
1696                 intel_prepare_page_flip(dev, intel_crtc->plane);
1697         }
1698 }
1699
1700 /* Called from drm generic code, passed 'crtc' which
1701  * we use as a pipe index
1702  */
1703 static int i915_enable_vblank(struct drm_device *dev, int pipe)
1704 {
1705         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1706         unsigned long irqflags;
1707
1708         if (!i915_pipe_enabled(dev, pipe))
1709                 return -EINVAL;
1710
1711         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1712         if (INTEL_INFO(dev)->gen >= 4)
1713                 i915_enable_pipestat(dev_priv, pipe,
1714                                      PIPE_START_VBLANK_INTERRUPT_ENABLE);
1715         else
1716                 i915_enable_pipestat(dev_priv, pipe,
1717                                      PIPE_VBLANK_INTERRUPT_ENABLE);
1718
1719         /* maintain vblank delivery even in deep C-states */
1720         if (dev_priv->info->gen == 3)
1721                 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
1722         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1723
1724         return 0;
1725 }
1726
1727 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
1728 {
1729         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1730         unsigned long irqflags;
1731         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1732                                                      DE_PIPE_VBLANK_ILK(pipe);
1733
1734         if (!i915_pipe_enabled(dev, pipe))
1735                 return -EINVAL;
1736
1737         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1738         ironlake_enable_display_irq(dev_priv, bit);
1739         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1740
1741         return 0;
1742 }
1743
1744 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
1745 {
1746         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1747         unsigned long irqflags;
1748         u32 imr;
1749
1750         if (!i915_pipe_enabled(dev, pipe))
1751                 return -EINVAL;
1752
1753         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1754         imr = I915_READ(VLV_IMR);
1755         if (pipe == 0)
1756                 imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
1757         else
1758                 imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1759         I915_WRITE(VLV_IMR, imr);
1760         i915_enable_pipestat(dev_priv, pipe,
1761                              PIPE_START_VBLANK_INTERRUPT_ENABLE);
1762         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1763
1764         return 0;
1765 }
1766
1767 /* Called from drm generic code, passed 'crtc' which
1768  * we use as a pipe index
1769  */
1770 static void i915_disable_vblank(struct drm_device *dev, int pipe)
1771 {
1772         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1773         unsigned long irqflags;
1774
1775         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1776         if (dev_priv->info->gen == 3)
1777                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
1778
1779         i915_disable_pipestat(dev_priv, pipe,
1780                               PIPE_VBLANK_INTERRUPT_ENABLE |
1781                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
1782         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1783 }
1784
1785 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
1786 {
1787         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1788         unsigned long irqflags;
1789         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1790                                                      DE_PIPE_VBLANK_ILK(pipe);
1791
1792         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1793         ironlake_disable_display_irq(dev_priv, bit);
1794         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1795 }
1796
1797 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
1798 {
1799         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1800         unsigned long irqflags;
1801         u32 imr;
1802
1803         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1804         i915_disable_pipestat(dev_priv, pipe,
1805                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
1806         imr = I915_READ(VLV_IMR);
1807         if (pipe == 0)
1808                 imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
1809         else
1810                 imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1811         I915_WRITE(VLV_IMR, imr);
1812         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1813 }
1814
1815 static u32
1816 ring_last_seqno(struct intel_ring_buffer *ring)
1817 {
1818         return list_entry(ring->request_list.prev,
1819                           struct drm_i915_gem_request, list)->seqno;
1820 }
1821
1822 static bool
1823 ring_idle(struct intel_ring_buffer *ring, u32 seqno)
1824 {
1825         return (list_empty(&ring->request_list) ||
1826                 i915_seqno_passed(seqno, ring_last_seqno(ring)));
1827 }
1828
1829 static struct intel_ring_buffer *
1830 semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
1831 {
1832         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1833         u32 cmd, ipehr, acthd, acthd_min;
1834
1835         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
1836         if ((ipehr & ~(0x3 << 16)) !=
1837             (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
1838                 return NULL;
1839
1840         /* ACTHD is likely pointing to the dword after the actual command,
1841          * so scan backwards until we find the MBOX.
1842          */
1843         acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
1844         acthd_min = max((int)acthd - 3 * 4, 0);
1845         do {
1846                 cmd = ioread32(ring->virtual_start + acthd);
1847                 if (cmd == ipehr)
1848                         break;
1849
1850                 acthd -= 4;
1851                 if (acthd < acthd_min)
1852                         return NULL;
1853         } while (1);
1854
1855         *seqno = ioread32(ring->virtual_start+acthd+4)+1;
1856         return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
1857 }
1858
1859 static int semaphore_passed(struct intel_ring_buffer *ring)
1860 {
1861         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1862         struct intel_ring_buffer *signaller;
1863         u32 seqno, ctl;
1864
1865         ring->hangcheck.deadlock = true;
1866
1867         signaller = semaphore_waits_for(ring, &seqno);
1868         if (signaller == NULL || signaller->hangcheck.deadlock)
1869                 return -1;
1870
1871         /* cursory check for an unkickable deadlock */
1872         ctl = I915_READ_CTL(signaller);
1873         if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
1874                 return -1;
1875
1876         return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
1877 }
1878
1879 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
1880 {
1881         struct intel_ring_buffer *ring;
1882         int i;
1883
1884         for_each_ring(ring, dev_priv, i)
1885                 ring->hangcheck.deadlock = false;
1886 }
1887
1888 static enum intel_ring_hangcheck_action
1889 ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
1890 {
1891         struct drm_device *dev = ring->dev;
1892         struct drm_i915_private *dev_priv = dev->dev_private;
1893         u32 tmp;
1894
1895         if (ring->hangcheck.acthd != acthd)
1896                 return HANGCHECK_ACTIVE;
1897
1898         if (IS_GEN2(dev))
1899                 return HANGCHECK_HUNG;
1900
1901         /* Is the chip hanging on a WAIT_FOR_EVENT?
1902          * If so we can simply poke the RB_WAIT bit
1903          * and break the hang. This should work on
1904          * all but the second generation chipsets.
1905          */
1906         tmp = I915_READ_CTL(ring);
1907         if (tmp & RING_WAIT) {
1908                 DRM_ERROR("Kicking stuck wait on %s\n",
1909                           ring->name);
1910                 I915_WRITE_CTL(ring, tmp);
1911                 return HANGCHECK_KICK;
1912         }
1913
1914         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
1915                 switch (semaphore_passed(ring)) {
1916                 default:
1917                         return HANGCHECK_HUNG;
1918                 case 1:
1919                         DRM_ERROR("Kicking stuck semaphore on %s\n",
1920                                   ring->name);
1921                         I915_WRITE_CTL(ring, tmp);
1922                         return HANGCHECK_KICK;
1923                 case 0:
1924                         return HANGCHECK_WAIT;
1925                 }
1926         }
1927
1928         return HANGCHECK_HUNG;
1929 }
1930
1931 /**
1932  * This is called when the chip hasn't reported back with completed
1933  * batchbuffers in a long time. We keep track per ring seqno progress and
1934  * if there are no progress, hangcheck score for that ring is increased.
1935  * Further, acthd is inspected to see if the ring is stuck. On stuck case
1936  * we kick the ring. If we see no progress on three subsequent calls
1937  * we assume chip is wedged and try to fix it by resetting the chip.
1938  */
1939 static void i915_hangcheck_elapsed(unsigned long data)
1940 {
1941         struct drm_device *dev = (struct drm_device *)data;
1942         drm_i915_private_t *dev_priv = dev->dev_private;
1943         struct intel_ring_buffer *ring;
1944         int i;
1945         int busy_count = 0, rings_hung = 0;
1946         bool stuck[I915_NUM_RINGS] = { 0 };
1947 #define BUSY 1
1948 #define KICK 5
1949 #define HUNG 20
1950 #define FIRE 30
1951
1952         if (!i915_enable_hangcheck)
1953                 return;
1954
1955         for_each_ring(ring, dev_priv, i) {
1956                 u32 seqno, acthd;
1957                 bool busy = true;
1958
1959                 semaphore_clear_deadlocks(dev_priv);
1960
1961                 seqno = ring->get_seqno(ring, false);
1962                 acthd = intel_ring_get_active_head(ring);
1963
1964                 if (ring->hangcheck.seqno == seqno) {
1965                         if (ring_idle(ring, seqno)) {
1966                                 if (waitqueue_active(&ring->irq_queue)) {
1967                                         /* Issue a wake-up to catch stuck h/w. */
1968                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
1969                                                   ring->name);
1970                                         wake_up_all(&ring->irq_queue);
1971                                         ring->hangcheck.score += HUNG;
1972                                 } else
1973                                         busy = false;
1974                         } else {
1975                                 /* We always increment the hangcheck score
1976                                  * if the ring is busy and still processing
1977                                  * the same request, so that no single request
1978                                  * can run indefinitely (such as a chain of
1979                                  * batches). The only time we do not increment
1980                                  * the hangcheck score on this ring, if this
1981                                  * ring is in a legitimate wait for another
1982                                  * ring. In that case the waiting ring is a
1983                                  * victim and we want to be sure we catch the
1984                                  * right culprit. Then every time we do kick
1985                                  * the ring, add a small increment to the
1986                                  * score so that we can catch a batch that is
1987                                  * being repeatedly kicked and so responsible
1988                                  * for stalling the machine.
1989                                  */
1990                                 ring->hangcheck.action = ring_stuck(ring,
1991                                                                     acthd);
1992
1993                                 switch (ring->hangcheck.action) {
1994                                 case HANGCHECK_WAIT:
1995                                         break;
1996                                 case HANGCHECK_ACTIVE:
1997                                         ring->hangcheck.score += BUSY;
1998                                         break;
1999                                 case HANGCHECK_KICK:
2000                                         ring->hangcheck.score += KICK;
2001                                         break;
2002                                 case HANGCHECK_HUNG:
2003                                         ring->hangcheck.score += HUNG;
2004                                         stuck[i] = true;
2005                                         break;
2006                                 }
2007                         }
2008                 } else {
2009                         /* Gradually reduce the count so that we catch DoS
2010                          * attempts across multiple batches.
2011                          */
2012                         if (ring->hangcheck.score > 0)
2013                                 ring->hangcheck.score--;
2014                 }
2015
2016                 ring->hangcheck.seqno = seqno;
2017                 ring->hangcheck.acthd = acthd;
2018                 busy_count += busy;
2019         }
2020
2021         for_each_ring(ring, dev_priv, i) {
2022                 if (ring->hangcheck.score > FIRE) {
2023                         DRM_ERROR("%s on %s\n",
2024                                   stuck[i] ? "stuck" : "no progress",
2025                                   ring->name);
2026                         rings_hung++;
2027                 }
2028         }
2029
2030         if (rings_hung)
2031                 return i915_handle_error(dev, true);
2032
2033         if (busy_count)
2034                 /* Reset timer case chip hangs without another request
2035                  * being added */
2036                 i915_queue_hangcheck(dev);
2037 }
2038
2039 void i915_queue_hangcheck(struct drm_device *dev)
2040 {
2041         struct drm_i915_private *dev_priv = dev->dev_private;
2042         if (!i915_enable_hangcheck)
2043                 return;
2044
2045         mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2046                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2047 }
2048
2049 static void ibx_irq_preinstall(struct drm_device *dev)
2050 {
2051         struct drm_i915_private *dev_priv = dev->dev_private;
2052
2053         if (HAS_PCH_NOP(dev))
2054                 return;
2055
2056         /* south display irq */
2057         I915_WRITE(SDEIMR, 0xffffffff);
2058         /*
2059          * SDEIER is also touched by the interrupt handler to work around missed
2060          * PCH interrupts. Hence we can't update it after the interrupt handler
2061          * is enabled - instead we unconditionally enable all PCH interrupt
2062          * sources here, but then only unmask them as needed with SDEIMR.
2063          */
2064         I915_WRITE(SDEIER, 0xffffffff);
2065         POSTING_READ(SDEIER);
2066 }
2067
2068 static void gen5_gt_irq_preinstall(struct drm_device *dev)
2069 {
2070         struct drm_i915_private *dev_priv = dev->dev_private;
2071
2072         /* and GT */
2073         I915_WRITE(GTIMR, 0xffffffff);
2074         I915_WRITE(GTIER, 0x0);
2075         POSTING_READ(GTIER);
2076
2077         if (INTEL_INFO(dev)->gen >= 6) {
2078                 /* and PM */
2079                 I915_WRITE(GEN6_PMIMR, 0xffffffff);
2080                 I915_WRITE(GEN6_PMIER, 0x0);
2081                 POSTING_READ(GEN6_PMIER);
2082         }
2083 }
2084
2085 /* drm_dma.h hooks
2086 */
2087 static void ironlake_irq_preinstall(struct drm_device *dev)
2088 {
2089         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2090
2091         atomic_set(&dev_priv->irq_received, 0);
2092
2093         I915_WRITE(HWSTAM, 0xeffe);
2094
2095         I915_WRITE(DEIMR, 0xffffffff);
2096         I915_WRITE(DEIER, 0x0);
2097         POSTING_READ(DEIER);
2098
2099         gen5_gt_irq_preinstall(dev);
2100
2101         ibx_irq_preinstall(dev);
2102 }
2103
2104 static void valleyview_irq_preinstall(struct drm_device *dev)
2105 {
2106         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2107         int pipe;
2108
2109         atomic_set(&dev_priv->irq_received, 0);
2110
2111         /* VLV magic */
2112         I915_WRITE(VLV_IMR, 0);
2113         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2114         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2115         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2116
2117         /* and GT */
2118         I915_WRITE(GTIIR, I915_READ(GTIIR));
2119         I915_WRITE(GTIIR, I915_READ(GTIIR));
2120
2121         gen5_gt_irq_preinstall(dev);
2122
2123         I915_WRITE(DPINVGTT, 0xff);
2124
2125         I915_WRITE(PORT_HOTPLUG_EN, 0);
2126         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2127         for_each_pipe(pipe)
2128                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2129         I915_WRITE(VLV_IIR, 0xffffffff);
2130         I915_WRITE(VLV_IMR, 0xffffffff);
2131         I915_WRITE(VLV_IER, 0x0);
2132         POSTING_READ(VLV_IER);
2133 }
2134
2135 static void ibx_hpd_irq_setup(struct drm_device *dev)
2136 {
2137         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2138         struct drm_mode_config *mode_config = &dev->mode_config;
2139         struct intel_encoder *intel_encoder;
2140         u32 hotplug_irqs, hotplug, enabled_irqs = 0;
2141
2142         if (HAS_PCH_IBX(dev)) {
2143                 hotplug_irqs = SDE_HOTPLUG_MASK;
2144                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2145                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2146                                 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
2147         } else {
2148                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
2149                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2150                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2151                                 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
2152         }
2153
2154         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
2155
2156         /*
2157          * Enable digital hotplug on the PCH, and configure the DP short pulse
2158          * duration to 2ms (which is the minimum in the Display Port spec)
2159          *
2160          * This register is the same on all known PCH chips.
2161          */
2162         hotplug = I915_READ(PCH_PORT_HOTPLUG);
2163         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2164         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2165         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2166         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2167         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2168 }
2169
2170 static void ibx_irq_postinstall(struct drm_device *dev)
2171 {
2172         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2173         u32 mask;
2174
2175         if (HAS_PCH_NOP(dev))
2176                 return;
2177
2178         if (HAS_PCH_IBX(dev)) {
2179                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER |
2180                        SDE_TRANSA_FIFO_UNDER | SDE_POISON;
2181         } else {
2182                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT;
2183
2184                 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2185         }
2186
2187         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2188         I915_WRITE(SDEIMR, ~mask);
2189 }
2190
2191 static void gen5_gt_irq_postinstall(struct drm_device *dev)
2192 {
2193         struct drm_i915_private *dev_priv = dev->dev_private;
2194         u32 pm_irqs, gt_irqs;
2195
2196         pm_irqs = gt_irqs = 0;
2197
2198         dev_priv->gt_irq_mask = ~0;
2199         if (HAS_L3_GPU_CACHE(dev)) {
2200                 /* L3 parity interrupt is always unmasked. */
2201                 dev_priv->gt_irq_mask = ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2202                 gt_irqs |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2203         }
2204
2205         gt_irqs |= GT_RENDER_USER_INTERRUPT;
2206         if (IS_GEN5(dev)) {
2207                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
2208                            ILK_BSD_USER_INTERRUPT;
2209         } else {
2210                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
2211         }
2212
2213         I915_WRITE(GTIIR, I915_READ(GTIIR));
2214         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2215         I915_WRITE(GTIER, gt_irqs);
2216         POSTING_READ(GTIER);
2217
2218         if (INTEL_INFO(dev)->gen >= 6) {
2219                 pm_irqs |= GEN6_PM_RPS_EVENTS;
2220
2221                 if (HAS_VEBOX(dev))
2222                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
2223
2224                 dev_priv->pm_irq_mask = 0xffffffff;
2225                 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
2226                 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
2227                 I915_WRITE(GEN6_PMIER, pm_irqs);
2228                 POSTING_READ(GEN6_PMIER);
2229         }
2230 }
2231
2232 static int ironlake_irq_postinstall(struct drm_device *dev)
2233 {
2234         unsigned long irqflags;
2235         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2236         u32 display_mask, extra_mask;
2237
2238         if (INTEL_INFO(dev)->gen >= 7) {
2239                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
2240                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
2241                                 DE_PLANEB_FLIP_DONE_IVB |
2242                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB |
2243                                 DE_ERR_INT_IVB);
2244                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
2245                               DE_PIPEA_VBLANK_IVB);
2246
2247                 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2248         } else {
2249                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
2250                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
2251                                 DE_AUX_CHANNEL_A | DE_PIPEB_FIFO_UNDERRUN |
2252                                 DE_PIPEA_FIFO_UNDERRUN | DE_POISON);
2253                 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT;
2254         }
2255
2256         dev_priv->irq_mask = ~display_mask;
2257
2258         /* should always can generate irq */
2259         I915_WRITE(DEIIR, I915_READ(DEIIR));
2260         I915_WRITE(DEIMR, dev_priv->irq_mask);
2261         I915_WRITE(DEIER, display_mask | extra_mask);
2262         POSTING_READ(DEIER);
2263
2264         gen5_gt_irq_postinstall(dev);
2265
2266         ibx_irq_postinstall(dev);
2267
2268         if (IS_IRONLAKE_M(dev)) {
2269                 /* Enable PCU event interrupts
2270                  *
2271                  * spinlocking not required here for correctness since interrupt
2272                  * setup is guaranteed to run in single-threaded context. But we
2273                  * need it to make the assert_spin_locked happy. */
2274                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2275                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
2276                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2277         }
2278
2279         return 0;
2280 }
2281
2282 static int valleyview_irq_postinstall(struct drm_device *dev)
2283 {
2284         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2285         u32 enable_mask;
2286         u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
2287         unsigned long irqflags;
2288
2289         enable_mask = I915_DISPLAY_PORT_INTERRUPT;
2290         enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2291                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2292                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2293                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2294
2295         /*
2296          *Leave vblank interrupts masked initially.  enable/disable will
2297          * toggle them based on usage.
2298          */
2299         dev_priv->irq_mask = (~enable_mask) |
2300                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2301                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2302
2303         I915_WRITE(PORT_HOTPLUG_EN, 0);
2304         POSTING_READ(PORT_HOTPLUG_EN);
2305
2306         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
2307         I915_WRITE(VLV_IER, enable_mask);
2308         I915_WRITE(VLV_IIR, 0xffffffff);
2309         I915_WRITE(PIPESTAT(0), 0xffff);
2310         I915_WRITE(PIPESTAT(1), 0xffff);
2311         POSTING_READ(VLV_IER);
2312
2313         /* Interrupt setup is already guaranteed to be single-threaded, this is
2314          * just to make the assert_spin_locked check happy. */
2315         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2316         i915_enable_pipestat(dev_priv, 0, pipestat_enable);
2317         i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
2318         i915_enable_pipestat(dev_priv, 1, pipestat_enable);
2319         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2320
2321         I915_WRITE(VLV_IIR, 0xffffffff);
2322         I915_WRITE(VLV_IIR, 0xffffffff);
2323
2324         gen5_gt_irq_postinstall(dev);
2325
2326         /* ack & enable invalid PTE error interrupts */
2327 #if 0 /* FIXME: add support to irq handler for checking these bits */
2328         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2329         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
2330 #endif
2331
2332         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
2333
2334         return 0;
2335 }
2336
2337 static void valleyview_irq_uninstall(struct drm_device *dev)
2338 {
2339         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2340         int pipe;
2341
2342         if (!dev_priv)
2343                 return;
2344
2345         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2346
2347         for_each_pipe(pipe)
2348                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2349
2350         I915_WRITE(HWSTAM, 0xffffffff);
2351         I915_WRITE(PORT_HOTPLUG_EN, 0);
2352         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2353         for_each_pipe(pipe)
2354                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2355         I915_WRITE(VLV_IIR, 0xffffffff);
2356         I915_WRITE(VLV_IMR, 0xffffffff);
2357         I915_WRITE(VLV_IER, 0x0);
2358         POSTING_READ(VLV_IER);
2359 }
2360
2361 static void ironlake_irq_uninstall(struct drm_device *dev)
2362 {
2363         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2364
2365         if (!dev_priv)
2366                 return;
2367
2368         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2369
2370         I915_WRITE(HWSTAM, 0xffffffff);
2371
2372         I915_WRITE(DEIMR, 0xffffffff);
2373         I915_WRITE(DEIER, 0x0);
2374         I915_WRITE(DEIIR, I915_READ(DEIIR));
2375         if (IS_GEN7(dev))
2376                 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2377
2378         I915_WRITE(GTIMR, 0xffffffff);
2379         I915_WRITE(GTIER, 0x0);
2380         I915_WRITE(GTIIR, I915_READ(GTIIR));
2381
2382         if (HAS_PCH_NOP(dev))
2383                 return;
2384
2385         I915_WRITE(SDEIMR, 0xffffffff);
2386         I915_WRITE(SDEIER, 0x0);
2387         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2388         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2389                 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2390 }
2391
2392 static void i8xx_irq_preinstall(struct drm_device * dev)
2393 {
2394         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2395         int pipe;
2396
2397         atomic_set(&dev_priv->irq_received, 0);
2398
2399         for_each_pipe(pipe)
2400                 I915_WRITE(PIPESTAT(pipe), 0);
2401         I915_WRITE16(IMR, 0xffff);
2402         I915_WRITE16(IER, 0x0);
2403         POSTING_READ16(IER);
2404 }
2405
2406 static int i8xx_irq_postinstall(struct drm_device *dev)
2407 {
2408         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2409
2410         I915_WRITE16(EMR,
2411                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2412
2413         /* Unmask the interrupts that we always want on. */
2414         dev_priv->irq_mask =
2415                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2416                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2417                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2418                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2419                   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2420         I915_WRITE16(IMR, dev_priv->irq_mask);
2421
2422         I915_WRITE16(IER,
2423                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2424                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2425                      I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2426                      I915_USER_INTERRUPT);
2427         POSTING_READ16(IER);
2428
2429         return 0;
2430 }
2431
2432 /*
2433  * Returns true when a page flip has completed.
2434  */
2435 static bool i8xx_handle_vblank(struct drm_device *dev,
2436                                int pipe, u16 iir)
2437 {
2438         drm_i915_private_t *dev_priv = dev->dev_private;
2439         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(pipe);
2440
2441         if (!drm_handle_vblank(dev, pipe))
2442                 return false;
2443
2444         if ((iir & flip_pending) == 0)
2445                 return false;
2446
2447         intel_prepare_page_flip(dev, pipe);
2448
2449         /* We detect FlipDone by looking for the change in PendingFlip from '1'
2450          * to '0' on the following vblank, i.e. IIR has the Pendingflip
2451          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2452          * the flip is completed (no longer pending). Since this doesn't raise
2453          * an interrupt per se, we watch for the change at vblank.
2454          */
2455         if (I915_READ16(ISR) & flip_pending)
2456                 return false;
2457
2458         intel_finish_page_flip(dev, pipe);
2459
2460         return true;
2461 }
2462
2463 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
2464 {
2465         struct drm_device *dev = (struct drm_device *) arg;
2466         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2467         u16 iir, new_iir;
2468         u32 pipe_stats[2];
2469         unsigned long irqflags;
2470         int pipe;
2471         u16 flip_mask =
2472                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2473                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2474
2475         atomic_inc(&dev_priv->irq_received);
2476
2477         iir = I915_READ16(IIR);
2478         if (iir == 0)
2479                 return IRQ_NONE;
2480
2481         while (iir & ~flip_mask) {
2482                 /* Can't rely on pipestat interrupt bit in iir as it might
2483                  * have been cleared after the pipestat interrupt was received.
2484                  * It doesn't set the bit in iir again, but it still produces
2485                  * interrupts (for non-MSI).
2486                  */
2487                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2488                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2489                         i915_handle_error(dev, false);
2490
2491                 for_each_pipe(pipe) {
2492                         int reg = PIPESTAT(pipe);
2493                         pipe_stats[pipe] = I915_READ(reg);
2494
2495                         /*
2496                          * Clear the PIPE*STAT regs before the IIR
2497                          */
2498                         if (pipe_stats[pipe] & 0x8000ffff) {
2499                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2500                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2501                                                          pipe_name(pipe));
2502                                 I915_WRITE(reg, pipe_stats[pipe]);
2503                         }
2504                 }
2505                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2506
2507                 I915_WRITE16(IIR, iir & ~flip_mask);
2508                 new_iir = I915_READ16(IIR); /* Flush posted writes */
2509
2510                 i915_update_dri1_breadcrumb(dev);
2511
2512                 if (iir & I915_USER_INTERRUPT)
2513                         notify_ring(dev, &dev_priv->ring[RCS]);
2514
2515                 if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
2516                     i8xx_handle_vblank(dev, 0, iir))
2517                         flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(0);
2518
2519                 if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
2520                     i8xx_handle_vblank(dev, 1, iir))
2521                         flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(1);
2522
2523                 iir = new_iir;
2524         }
2525
2526         return IRQ_HANDLED;
2527 }
2528
2529 static void i8xx_irq_uninstall(struct drm_device * dev)
2530 {
2531         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2532         int pipe;
2533
2534         for_each_pipe(pipe) {
2535                 /* Clear enable bits; then clear status bits */
2536                 I915_WRITE(PIPESTAT(pipe), 0);
2537                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2538         }
2539         I915_WRITE16(IMR, 0xffff);
2540         I915_WRITE16(IER, 0x0);
2541         I915_WRITE16(IIR, I915_READ16(IIR));
2542 }
2543
2544 static void i915_irq_preinstall(struct drm_device * dev)
2545 {
2546         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2547         int pipe;
2548
2549         atomic_set(&dev_priv->irq_received, 0);
2550
2551         if (I915_HAS_HOTPLUG(dev)) {
2552                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2553                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2554         }
2555
2556         I915_WRITE16(HWSTAM, 0xeffe);
2557         for_each_pipe(pipe)
2558                 I915_WRITE(PIPESTAT(pipe), 0);
2559         I915_WRITE(IMR, 0xffffffff);
2560         I915_WRITE(IER, 0x0);
2561         POSTING_READ(IER);
2562 }
2563
2564 static int i915_irq_postinstall(struct drm_device *dev)
2565 {
2566         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2567         u32 enable_mask;
2568
2569         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2570
2571         /* Unmask the interrupts that we always want on. */
2572         dev_priv->irq_mask =
2573                 ~(I915_ASLE_INTERRUPT |
2574                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2575                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2576                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2577                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2578                   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2579
2580         enable_mask =
2581                 I915_ASLE_INTERRUPT |
2582                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2583                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2584                 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2585                 I915_USER_INTERRUPT;
2586
2587         if (I915_HAS_HOTPLUG(dev)) {
2588                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2589                 POSTING_READ(PORT_HOTPLUG_EN);
2590
2591                 /* Enable in IER... */
2592                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
2593                 /* and unmask in IMR */
2594                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
2595         }
2596
2597         I915_WRITE(IMR, dev_priv->irq_mask);
2598         I915_WRITE(IER, enable_mask);
2599         POSTING_READ(IER);
2600
2601         i915_enable_asle_pipestat(dev);
2602
2603         return 0;
2604 }
2605
2606 /*
2607  * Returns true when a page flip has completed.
2608  */
2609 static bool i915_handle_vblank(struct drm_device *dev,
2610                                int plane, int pipe, u32 iir)
2611 {
2612         drm_i915_private_t *dev_priv = dev->dev_private;
2613         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
2614
2615         if (!drm_handle_vblank(dev, pipe))
2616                 return false;
2617
2618         if ((iir & flip_pending) == 0)
2619                 return false;
2620
2621         intel_prepare_page_flip(dev, plane);
2622
2623         /* We detect FlipDone by looking for the change in PendingFlip from '1'
2624          * to '0' on the following vblank, i.e. IIR has the Pendingflip
2625          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2626          * the flip is completed (no longer pending). Since this doesn't raise
2627          * an interrupt per se, we watch for the change at vblank.
2628          */
2629         if (I915_READ(ISR) & flip_pending)
2630                 return false;
2631
2632         intel_finish_page_flip(dev, pipe);
2633
2634         return true;
2635 }
2636
2637 static irqreturn_t i915_irq_handler(int irq, void *arg)
2638 {
2639         struct drm_device *dev = (struct drm_device *) arg;
2640         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2641         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
2642         unsigned long irqflags;
2643         u32 flip_mask =
2644                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2645                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2646         int pipe, ret = IRQ_NONE;
2647
2648         atomic_inc(&dev_priv->irq_received);
2649
2650         iir = I915_READ(IIR);
2651         do {
2652                 bool irq_received = (iir & ~flip_mask) != 0;
2653                 bool blc_event = false;
2654
2655                 /* Can't rely on pipestat interrupt bit in iir as it might
2656                  * have been cleared after the pipestat interrupt was received.
2657                  * It doesn't set the bit in iir again, but it still produces
2658                  * interrupts (for non-MSI).
2659                  */
2660                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2661                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2662                         i915_handle_error(dev, false);
2663
2664                 for_each_pipe(pipe) {
2665                         int reg = PIPESTAT(pipe);
2666                         pipe_stats[pipe] = I915_READ(reg);
2667
2668                         /* Clear the PIPE*STAT regs before the IIR */
2669                         if (pipe_stats[pipe] & 0x8000ffff) {
2670                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2671                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2672                                                          pipe_name(pipe));
2673                                 I915_WRITE(reg, pipe_stats[pipe]);
2674                                 irq_received = true;
2675                         }
2676                 }
2677                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2678
2679                 if (!irq_received)
2680                         break;
2681
2682                 /* Consume port.  Then clear IIR or we'll miss events */
2683                 if ((I915_HAS_HOTPLUG(dev)) &&
2684                     (iir & I915_DISPLAY_PORT_INTERRUPT)) {
2685                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2686                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
2687
2688                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2689                                   hotplug_status);
2690
2691                         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
2692
2693                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2694                         POSTING_READ(PORT_HOTPLUG_STAT);
2695                 }
2696
2697                 I915_WRITE(IIR, iir & ~flip_mask);
2698                 new_iir = I915_READ(IIR); /* Flush posted writes */
2699
2700                 if (iir & I915_USER_INTERRUPT)
2701                         notify_ring(dev, &dev_priv->ring[RCS]);
2702
2703                 for_each_pipe(pipe) {
2704                         int plane = pipe;
2705                         if (IS_MOBILE(dev))
2706                                 plane = !plane;
2707
2708                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
2709                             i915_handle_vblank(dev, plane, pipe, iir))
2710                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
2711
2712                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2713                                 blc_event = true;
2714                 }
2715
2716                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2717                         intel_opregion_asle_intr(dev);
2718
2719                 /* With MSI, interrupts are only generated when iir
2720                  * transitions from zero to nonzero.  If another bit got
2721                  * set while we were handling the existing iir bits, then
2722                  * we would never get another interrupt.
2723                  *
2724                  * This is fine on non-MSI as well, as if we hit this path
2725                  * we avoid exiting the interrupt handler only to generate
2726                  * another one.
2727                  *
2728                  * Note that for MSI this could cause a stray interrupt report
2729                  * if an interrupt landed in the time between writing IIR and
2730                  * the posting read.  This should be rare enough to never
2731                  * trigger the 99% of 100,000 interrupts test for disabling
2732                  * stray interrupts.
2733                  */
2734                 ret = IRQ_HANDLED;
2735                 iir = new_iir;
2736         } while (iir & ~flip_mask);
2737
2738         i915_update_dri1_breadcrumb(dev);
2739
2740         return ret;
2741 }
2742
2743 static void i915_irq_uninstall(struct drm_device * dev)
2744 {
2745         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2746         int pipe;
2747
2748         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2749
2750         if (I915_HAS_HOTPLUG(dev)) {
2751                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2752                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2753         }
2754
2755         I915_WRITE16(HWSTAM, 0xffff);
2756         for_each_pipe(pipe) {
2757                 /* Clear enable bits; then clear status bits */
2758                 I915_WRITE(PIPESTAT(pipe), 0);
2759                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2760         }
2761         I915_WRITE(IMR, 0xffffffff);
2762         I915_WRITE(IER, 0x0);
2763
2764         I915_WRITE(IIR, I915_READ(IIR));
2765 }
2766
2767 static void i965_irq_preinstall(struct drm_device * dev)
2768 {
2769         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2770         int pipe;
2771
2772         atomic_set(&dev_priv->irq_received, 0);
2773
2774         I915_WRITE(PORT_HOTPLUG_EN, 0);
2775         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2776
2777         I915_WRITE(HWSTAM, 0xeffe);
2778         for_each_pipe(pipe)
2779                 I915_WRITE(PIPESTAT(pipe), 0);
2780         I915_WRITE(IMR, 0xffffffff);
2781         I915_WRITE(IER, 0x0);
2782         POSTING_READ(IER);
2783 }
2784
2785 static int i965_irq_postinstall(struct drm_device *dev)
2786 {
2787         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2788         u32 enable_mask;
2789         u32 error_mask;
2790         unsigned long irqflags;
2791
2792         /* Unmask the interrupts that we always want on. */
2793         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
2794                                I915_DISPLAY_PORT_INTERRUPT |
2795                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2796                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2797                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2798                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2799                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2800
2801         enable_mask = ~dev_priv->irq_mask;
2802         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2803                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
2804         enable_mask |= I915_USER_INTERRUPT;
2805
2806         if (IS_G4X(dev))
2807                 enable_mask |= I915_BSD_USER_INTERRUPT;
2808
2809         /* Interrupt setup is already guaranteed to be single-threaded, this is
2810          * just to make the assert_spin_locked check happy. */
2811         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2812         i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
2813         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2814
2815         /*
2816          * Enable some error detection, note the instruction error mask
2817          * bit is reserved, so we leave it masked.
2818          */
2819         if (IS_G4X(dev)) {
2820                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
2821                                GM45_ERROR_MEM_PRIV |
2822                                GM45_ERROR_CP_PRIV |
2823                                I915_ERROR_MEMORY_REFRESH);
2824         } else {
2825                 error_mask = ~(I915_ERROR_PAGE_TABLE |
2826                                I915_ERROR_MEMORY_REFRESH);
2827         }
2828         I915_WRITE(EMR, error_mask);
2829
2830         I915_WRITE(IMR, dev_priv->irq_mask);
2831         I915_WRITE(IER, enable_mask);
2832         POSTING_READ(IER);
2833
2834         I915_WRITE(PORT_HOTPLUG_EN, 0);
2835         POSTING_READ(PORT_HOTPLUG_EN);
2836
2837         i915_enable_asle_pipestat(dev);
2838
2839         return 0;
2840 }
2841
2842 static void i915_hpd_irq_setup(struct drm_device *dev)
2843 {
2844         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2845         struct drm_mode_config *mode_config = &dev->mode_config;
2846         struct intel_encoder *intel_encoder;
2847         u32 hotplug_en;
2848
2849         assert_spin_locked(&dev_priv->irq_lock);
2850
2851         if (I915_HAS_HOTPLUG(dev)) {
2852                 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
2853                 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
2854                 /* Note HDMI and DP share hotplug bits */
2855                 /* enable bits are the same for all generations */
2856                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2857                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2858                                 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
2859                 /* Programming the CRT detection parameters tends
2860                    to generate a spurious hotplug event about three
2861                    seconds later.  So just do it once.
2862                 */
2863                 if (IS_G4X(dev))
2864                         hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
2865                 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
2866                 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
2867
2868                 /* Ignore TV since it's buggy */
2869                 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
2870         }
2871 }
2872
2873 static irqreturn_t i965_irq_handler(int irq, void *arg)
2874 {
2875         struct drm_device *dev = (struct drm_device *) arg;
2876         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2877         u32 iir, new_iir;
2878         u32 pipe_stats[I915_MAX_PIPES];
2879         unsigned long irqflags;
2880         int irq_received;
2881         int ret = IRQ_NONE, pipe;
2882         u32 flip_mask =
2883                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2884                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2885
2886         atomic_inc(&dev_priv->irq_received);
2887
2888         iir = I915_READ(IIR);
2889
2890         for (;;) {
2891                 bool blc_event = false;
2892
2893                 irq_received = (iir & ~flip_mask) != 0;
2894
2895                 /* Can't rely on pipestat interrupt bit in iir as it might
2896                  * have been cleared after the pipestat interrupt was received.
2897                  * It doesn't set the bit in iir again, but it still produces
2898                  * interrupts (for non-MSI).
2899                  */
2900                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2901                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2902                         i915_handle_error(dev, false);
2903
2904                 for_each_pipe(pipe) {
2905                         int reg = PIPESTAT(pipe);
2906                         pipe_stats[pipe] = I915_READ(reg);
2907
2908                         /*
2909                          * Clear the PIPE*STAT regs before the IIR
2910                          */
2911                         if (pipe_stats[pipe] & 0x8000ffff) {
2912                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2913                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2914                                                          pipe_name(pipe));
2915                                 I915_WRITE(reg, pipe_stats[pipe]);
2916                                 irq_received = 1;
2917                         }
2918                 }
2919                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2920
2921                 if (!irq_received)
2922                         break;
2923
2924                 ret = IRQ_HANDLED;
2925
2926                 /* Consume port.  Then clear IIR or we'll miss events */
2927                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
2928                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2929                         u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ?
2930                                                                   HOTPLUG_INT_STATUS_G4X :
2931                                                                   HOTPLUG_INT_STATUS_I915);
2932
2933                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2934                                   hotplug_status);
2935
2936                         intel_hpd_irq_handler(dev, hotplug_trigger,
2937                                               IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i915);
2938
2939                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2940                         I915_READ(PORT_HOTPLUG_STAT);
2941                 }
2942
2943                 I915_WRITE(IIR, iir & ~flip_mask);
2944                 new_iir = I915_READ(IIR); /* Flush posted writes */
2945
2946                 if (iir & I915_USER_INTERRUPT)
2947                         notify_ring(dev, &dev_priv->ring[RCS]);
2948                 if (iir & I915_BSD_USER_INTERRUPT)
2949                         notify_ring(dev, &dev_priv->ring[VCS]);
2950
2951                 for_each_pipe(pipe) {
2952                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
2953                             i915_handle_vblank(dev, pipe, pipe, iir))
2954                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
2955
2956                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2957                                 blc_event = true;
2958                 }
2959
2960
2961                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2962                         intel_opregion_asle_intr(dev);
2963
2964                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
2965                         gmbus_irq_handler(dev);
2966
2967                 /* With MSI, interrupts are only generated when iir
2968                  * transitions from zero to nonzero.  If another bit got
2969                  * set while we were handling the existing iir bits, then
2970                  * we would never get another interrupt.
2971                  *
2972                  * This is fine on non-MSI as well, as if we hit this path
2973                  * we avoid exiting the interrupt handler only to generate
2974                  * another one.
2975                  *
2976                  * Note that for MSI this could cause a stray interrupt report
2977                  * if an interrupt landed in the time between writing IIR and
2978                  * the posting read.  This should be rare enough to never
2979                  * trigger the 99% of 100,000 interrupts test for disabling
2980                  * stray interrupts.
2981                  */
2982                 iir = new_iir;
2983         }
2984
2985         i915_update_dri1_breadcrumb(dev);
2986
2987         return ret;
2988 }
2989
2990 static void i965_irq_uninstall(struct drm_device * dev)
2991 {
2992         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2993         int pipe;
2994
2995         if (!dev_priv)
2996                 return;
2997
2998         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2999
3000         I915_WRITE(PORT_HOTPLUG_EN, 0);
3001         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3002
3003         I915_WRITE(HWSTAM, 0xffffffff);
3004         for_each_pipe(pipe)
3005                 I915_WRITE(PIPESTAT(pipe), 0);
3006         I915_WRITE(IMR, 0xffffffff);
3007         I915_WRITE(IER, 0x0);
3008
3009         for_each_pipe(pipe)
3010                 I915_WRITE(PIPESTAT(pipe),
3011                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
3012         I915_WRITE(IIR, I915_READ(IIR));
3013 }
3014
3015 static void i915_reenable_hotplug_timer_func(unsigned long data)
3016 {
3017         drm_i915_private_t *dev_priv = (drm_i915_private_t *)data;
3018         struct drm_device *dev = dev_priv->dev;
3019         struct drm_mode_config *mode_config = &dev->mode_config;
3020         unsigned long irqflags;
3021         int i;
3022
3023         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3024         for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
3025                 struct drm_connector *connector;
3026
3027                 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
3028                         continue;
3029
3030                 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3031
3032                 list_for_each_entry(connector, &mode_config->connector_list, head) {
3033                         struct intel_connector *intel_connector = to_intel_connector(connector);
3034
3035                         if (intel_connector->encoder->hpd_pin == i) {
3036                                 if (connector->polled != intel_connector->polled)
3037                                         DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
3038                                                          drm_get_connector_name(connector));
3039                                 connector->polled = intel_connector->polled;
3040                                 if (!connector->polled)
3041                                         connector->polled = DRM_CONNECTOR_POLL_HPD;
3042                         }
3043                 }
3044         }
3045         if (dev_priv->display.hpd_irq_setup)
3046                 dev_priv->display.hpd_irq_setup(dev);
3047         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3048 }
3049
3050 void intel_irq_init(struct drm_device *dev)
3051 {
3052         struct drm_i915_private *dev_priv = dev->dev_private;
3053
3054         INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
3055         INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
3056         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
3057         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
3058
3059         setup_timer(&dev_priv->gpu_error.hangcheck_timer,
3060                     i915_hangcheck_elapsed,
3061                     (unsigned long) dev);
3062         setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func,
3063                     (unsigned long) dev_priv);
3064
3065         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
3066
3067         dev->driver->get_vblank_counter = i915_get_vblank_counter;
3068         dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
3069         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
3070                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
3071                 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
3072         }
3073
3074         if (drm_core_check_feature(dev, DRIVER_MODESET))
3075                 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
3076         else
3077                 dev->driver->get_vblank_timestamp = NULL;
3078         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3079
3080         if (IS_VALLEYVIEW(dev)) {
3081                 dev->driver->irq_handler = valleyview_irq_handler;
3082                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
3083                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
3084                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
3085                 dev->driver->enable_vblank = valleyview_enable_vblank;
3086                 dev->driver->disable_vblank = valleyview_disable_vblank;
3087                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3088         } else if (HAS_PCH_SPLIT(dev)) {
3089                 dev->driver->irq_handler = ironlake_irq_handler;
3090                 dev->driver->irq_preinstall = ironlake_irq_preinstall;
3091                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
3092                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3093                 dev->driver->enable_vblank = ironlake_enable_vblank;
3094                 dev->driver->disable_vblank = ironlake_disable_vblank;
3095                 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
3096         } else {
3097                 if (INTEL_INFO(dev)->gen == 2) {
3098                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
3099                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
3100                         dev->driver->irq_handler = i8xx_irq_handler;
3101                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
3102                 } else if (INTEL_INFO(dev)->gen == 3) {
3103                         dev->driver->irq_preinstall = i915_irq_preinstall;
3104                         dev->driver->irq_postinstall = i915_irq_postinstall;
3105                         dev->driver->irq_uninstall = i915_irq_uninstall;
3106                         dev->driver->irq_handler = i915_irq_handler;
3107                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3108                 } else {
3109                         dev->driver->irq_preinstall = i965_irq_preinstall;
3110                         dev->driver->irq_postinstall = i965_irq_postinstall;
3111                         dev->driver->irq_uninstall = i965_irq_uninstall;
3112                         dev->driver->irq_handler = i965_irq_handler;
3113                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3114                 }
3115                 dev->driver->enable_vblank = i915_enable_vblank;
3116                 dev->driver->disable_vblank = i915_disable_vblank;
3117         }
3118 }
3119
3120 void intel_hpd_init(struct drm_device *dev)
3121 {
3122         struct drm_i915_private *dev_priv = dev->dev_private;
3123         struct drm_mode_config *mode_config = &dev->mode_config;
3124         struct drm_connector *connector;
3125         unsigned long irqflags;
3126         int i;
3127
3128         for (i = 1; i < HPD_NUM_PINS; i++) {
3129                 dev_priv->hpd_stats[i].hpd_cnt = 0;
3130                 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3131         }
3132         list_for_each_entry(connector, &mode_config->connector_list, head) {
3133                 struct intel_connector *intel_connector = to_intel_connector(connector);
3134                 connector->polled = intel_connector->polled;
3135                 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
3136                         connector->polled = DRM_CONNECTOR_POLL_HPD;
3137         }
3138
3139         /* Interrupt setup is already guaranteed to be single-threaded, this is
3140          * just to make the assert_spin_locked checks happy. */
3141         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3142         if (dev_priv->display.hpd_irq_setup)
3143                 dev_priv->display.hpd_irq_setup(dev);
3144         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3145 }