drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/cpufreq.h>
  28 #include <linux/module.h>
  29 #include <linux/input.h>
  30 #include <linux/i2c.h>
  31 #include <linux/kernel.h>
  32 #include <linux/slab.h>
  33 #include <linux/vgaarb.h>
  34 #include <drm/drm_edid.h>
  35 #include "drmP.h"
  36 #include "intel_drv.h"
  37 #include "i915_drm.h"
  38 #include "i915_drv.h"
  39 #include "i915_trace.h"
  40 #include "drm_dp_helper.h"
  41
  42 #include "drm_crtc_helper.h"
  43
  44 #define HAS_eDP (intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
  45
  46 bool intel_pipe_has_type(struct drm_crtc *crtc, int type);
  47 static void intel_update_watermarks(struct drm_device *dev);
  48 static void intel_increase_pllclock(struct drm_crtc *crtc);
  49 static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
  50
  51 typedef struct {
  52         /* given values */
  53         int n;
  54         int m1, m2;
  55         int p1, p2;
  56         /* derived values */
  57         int     dot;
  58         int     vco;
  59         int     m;
  60         int     p;
  61 } intel_clock_t;
  62
  63 typedef struct {
  64         int     min, max;
  65 } intel_range_t;
  66
  67 typedef struct {
  68         int     dot_limit;
  69         int     p2_slow, p2_fast;
  70 } intel_p2_t;
  71
  72 #define INTEL_P2_NUM                  2
  73 typedef struct intel_limit intel_limit_t;
  74 struct intel_limit {
  75         intel_range_t   dot, vco, n, m, m1, m2, p, p1;
  76         intel_p2_t          p2;
  77         bool (* find_pll)(const intel_limit_t *, struct drm_crtc *,
  78                         int, int, intel_clock_t *);
  79 };
  80
  81 /* FDI */
  82 #define IRONLAKE_FDI_FREQ               2700000 /* in kHz for mode->clock */
  83
  84 static bool
  85 intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
  86                     int target, int refclk, intel_clock_t *best_clock);
  87 static bool
  88 intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
  89                         int target, int refclk, intel_clock_t *best_clock);
  90
  91 static bool
  92 intel_find_pll_g4x_dp(const intel_limit_t *, struct drm_crtc *crtc,
  93                       int target, int refclk, intel_clock_t *best_clock);
  94 static bool
  95 intel_find_pll_ironlake_dp(const intel_limit_t *, struct drm_crtc *crtc,
  96                            int target, int refclk, intel_clock_t *best_clock);
  97
  98 static inline u32 /* units of 100MHz */
  99 intel_fdi_link_freq(struct drm_device *dev)
 100 {
 101         if (IS_GEN5(dev)) {
 102                 struct drm_i915_private *dev_priv = dev->dev_private;
 103                 return (I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2;
 104         } else
 105                 return 27;
 106 }
 107
 108 static const intel_limit_t intel_limits_i8xx_dvo = {
 109         .dot = { .min = 25000, .max = 350000 },
 110         .vco = { .min = 930000, .max = 1400000 },
 111         .n = { .min = 3, .max = 16 },
 112         .m = { .min = 96, .max = 140 },
 113         .m1 = { .min = 18, .max = 26 },
 114         .m2 = { .min = 6, .max = 16 },
 115         .p = { .min = 4, .max = 128 },
 116         .p1 = { .min = 2, .max = 33 },
 117         .p2 = { .dot_limit = 165000,
 118                 .p2_slow = 4, .p2_fast = 2 },
 119         .find_pll = intel_find_best_PLL,
 120 };
 121
 122 static const intel_limit_t intel_limits_i8xx_lvds = {
 123         .dot = { .min = 25000, .max = 350000 },
 124         .vco = { .min = 930000, .max = 1400000 },
 125         .n = { .min = 3, .max = 16 },
 126         .m = { .min = 96, .max = 140 },
 127         .m1 = { .min = 18, .max = 26 },
 128         .m2 = { .min = 6, .max = 16 },
 129         .p = { .min = 4, .max = 128 },
 130         .p1 = { .min = 1, .max = 6 },
 131         .p2 = { .dot_limit = 165000,
 132                 .p2_slow = 14, .p2_fast = 7 },
 133         .find_pll = intel_find_best_PLL,
 134 };
 135
 136 static const intel_limit_t intel_limits_i9xx_sdvo = {
 137         .dot = { .min = 20000, .max = 400000 },
 138         .vco = { .min = 1400000, .max = 2800000 },
 139         .n = { .min = 1, .max = 6 },
 140         .m = { .min = 70, .max = 120 },
 141         .m1 = { .min = 10, .max = 22 },
 142         .m2 = { .min = 5, .max = 9 },
 143         .p = { .min = 5, .max = 80 },
 144         .p1 = { .min = 1, .max = 8 },
 145         .p2 = { .dot_limit = 200000,
 146                 .p2_slow = 10, .p2_fast = 5 },
 147         .find_pll = intel_find_best_PLL,
 148 };
 149
 150 static const intel_limit_t intel_limits_i9xx_lvds = {
 151         .dot = { .min = 20000, .max = 400000 },
 152         .vco = { .min = 1400000, .max = 2800000 },
 153         .n = { .min = 1, .max = 6 },
 154         .m = { .min = 70, .max = 120 },
 155         .m1 = { .min = 10, .max = 22 },
 156         .m2 = { .min = 5, .max = 9 },
 157         .p = { .min = 7, .max = 98 },
 158         .p1 = { .min = 1, .max = 8 },
 159         .p2 = { .dot_limit = 112000,
 160                 .p2_slow = 14, .p2_fast = 7 },
 161         .find_pll = intel_find_best_PLL,
 162 };
 163
 164
 165 static const intel_limit_t intel_limits_g4x_sdvo = {
 166         .dot = { .min = 25000, .max = 270000 },
 167         .vco = { .min = 1750000, .max = 3500000},
 168         .n = { .min = 1, .max = 4 },
 169         .m = { .min = 104, .max = 138 },
 170         .m1 = { .min = 17, .max = 23 },
 171         .m2 = { .min = 5, .max = 11 },
 172         .p = { .min = 10, .max = 30 },
 173         .p1 = { .min = 1, .max = 3},
 174         .p2 = { .dot_limit = 270000,
 175                 .p2_slow = 10,
 176                 .p2_fast = 10
 177         },
 178         .find_pll = intel_g4x_find_best_PLL,
 179 };
 180
 181 static const intel_limit_t intel_limits_g4x_hdmi = {
 182         .dot = { .min = 22000, .max = 400000 },
 183         .vco = { .min = 1750000, .max = 3500000},
 184         .n = { .min = 1, .max = 4 },
 185         .m = { .min = 104, .max = 138 },
 186         .m1 = { .min = 16, .max = 23 },
 187         .m2 = { .min = 5, .max = 11 },
 188         .p = { .min = 5, .max = 80 },
 189         .p1 = { .min = 1, .max = 8},
 190         .p2 = { .dot_limit = 165000,
 191                 .p2_slow = 10, .p2_fast = 5 },
 192         .find_pll = intel_g4x_find_best_PLL,
 193 };
 194
 195 static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
 196         .dot = { .min = 20000, .max = 115000 },
 197         .vco = { .min = 1750000, .max = 3500000 },
 198         .n = { .min = 1, .max = 3 },
 199         .m = { .min = 104, .max = 138 },
 200         .m1 = { .min = 17, .max = 23 },
 201         .m2 = { .min = 5, .max = 11 },
 202         .p = { .min = 28, .max = 112 },
 203         .p1 = { .min = 2, .max = 8 },
 204         .p2 = { .dot_limit = 0,
 205                 .p2_slow = 14, .p2_fast = 14
 206         },
 207         .find_pll = intel_g4x_find_best_PLL,
 208 };
 209
 210 static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
 211         .dot = { .min = 80000, .max = 224000 },
 212         .vco = { .min = 1750000, .max = 3500000 },
 213         .n = { .min = 1, .max = 3 },
 214         .m = { .min = 104, .max = 138 },
 215         .m1 = { .min = 17, .max = 23 },
 216         .m2 = { .min = 5, .max = 11 },
 217         .p = { .min = 14, .max = 42 },
 218         .p1 = { .min = 2, .max = 6 },
 219         .p2 = { .dot_limit = 0,
 220                 .p2_slow = 7, .p2_fast = 7
 221         },
 222         .find_pll = intel_g4x_find_best_PLL,
 223 };
 224
 225 static const intel_limit_t intel_limits_g4x_display_port = {
 226         .dot = { .min = 161670, .max = 227000 },
 227         .vco = { .min = 1750000, .max = 3500000},
 228         .n = { .min = 1, .max = 2 },
 229         .m = { .min = 97, .max = 108 },
 230         .m1 = { .min = 0x10, .max = 0x12 },
 231         .m2 = { .min = 0x05, .max = 0x06 },
 232         .p = { .min = 10, .max = 20 },
 233         .p1 = { .min = 1, .max = 2},
 234         .p2 = { .dot_limit = 0,
 235                 .p2_slow = 10, .p2_fast = 10 },
 236         .find_pll = intel_find_pll_g4x_dp,
 237 };
 238
 239 static const intel_limit_t intel_limits_pineview_sdvo = {
 240         .dot = { .min = 20000, .max = 400000},
 241         .vco = { .min = 1700000, .max = 3500000 },
 242         /* Pineview's Ncounter is a ring counter */
 243         .n = { .min = 3, .max = 6 },
 244         .m = { .min = 2, .max = 256 },
 245         /* Pineview only has one combined m divider, which we treat as m2. */
 246         .m1 = { .min = 0, .max = 0 },
 247         .m2 = { .min = 0, .max = 254 },
 248         .p = { .min = 5, .max = 80 },
 249         .p1 = { .min = 1, .max = 8 },
 250         .p2 = { .dot_limit = 200000,
 251                 .p2_slow = 10, .p2_fast = 5 },
 252         .find_pll = intel_find_best_PLL,
 253 };
 254
 255 static const intel_limit_t intel_limits_pineview_lvds = {
 256         .dot = { .min = 20000, .max = 400000 },
 257         .vco = { .min = 1700000, .max = 3500000 },
 258         .n = { .min = 3, .max = 6 },
 259         .m = { .min = 2, .max = 256 },
 260         .m1 = { .min = 0, .max = 0 },
 261         .m2 = { .min = 0, .max = 254 },
 262         .p = { .min = 7, .max = 112 },
 263         .p1 = { .min = 1, .max = 8 },
 264         .p2 = { .dot_limit = 112000,
 265                 .p2_slow = 14, .p2_fast = 14 },
 266         .find_pll = intel_find_best_PLL,
 267 };
 268
 269 /* Ironlake / Sandybridge
 270  *
 271  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 272  * the range value for them is (actual_value - 2).
 273  */
 274 static const intel_limit_t intel_limits_ironlake_dac = {
 275         .dot = { .min = 25000, .max = 350000 },
 276         .vco = { .min = 1760000, .max = 3510000 },
 277         .n = { .min = 1, .max = 5 },
 278         .m = { .min = 79, .max = 127 },
 279         .m1 = { .min = 12, .max = 22 },
 280         .m2 = { .min = 5, .max = 9 },
 281         .p = { .min = 5, .max = 80 },
 282         .p1 = { .min = 1, .max = 8 },
 283         .p2 = { .dot_limit = 225000,
 284                 .p2_slow = 10, .p2_fast = 5 },
 285         .find_pll = intel_g4x_find_best_PLL,
 286 };
 287
 288 static const intel_limit_t intel_limits_ironlake_single_lvds = {
 289         .dot = { .min = 25000, .max = 350000 },
 290         .vco = { .min = 1760000, .max = 3510000 },
 291         .n = { .min = 1, .max = 3 },
 292         .m = { .min = 79, .max = 118 },
 293         .m1 = { .min = 12, .max = 22 },
 294         .m2 = { .min = 5, .max = 9 },
 295         .p = { .min = 28, .max = 112 },
 296         .p1 = { .min = 2, .max = 8 },
 297         .p2 = { .dot_limit = 225000,
 298                 .p2_slow = 14, .p2_fast = 14 },
 299         .find_pll = intel_g4x_find_best_PLL,
 300 };
 301
 302 static const intel_limit_t intel_limits_ironlake_dual_lvds = {
 303         .dot = { .min = 25000, .max = 350000 },
 304         .vco = { .min = 1760000, .max = 3510000 },
 305         .n = { .min = 1, .max = 3 },
 306         .m = { .min = 79, .max = 127 },
 307         .m1 = { .min = 12, .max = 22 },
 308         .m2 = { .min = 5, .max = 9 },
 309         .p = { .min = 14, .max = 56 },
 310         .p1 = { .min = 2, .max = 8 },
 311         .p2 = { .dot_limit = 225000,
 312                 .p2_slow = 7, .p2_fast = 7 },
 313         .find_pll = intel_g4x_find_best_PLL,
 314 };
 315
 316 /* LVDS 100mhz refclk limits. */
 317 static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
 318         .dot = { .min = 25000, .max = 350000 },
 319         .vco = { .min = 1760000, .max = 3510000 },
 320         .n = { .min = 1, .max = 2 },
 321         .m = { .min = 79, .max = 126 },
 322         .m1 = { .min = 12, .max = 22 },
 323         .m2 = { .min = 5, .max = 9 },
 324         .p = { .min = 28, .max = 112 },
 325         .p1 = { .min = 2, .max = 8 },
 326         .p2 = { .dot_limit = 225000,
 327                 .p2_slow = 14, .p2_fast = 14 },
 328         .find_pll = intel_g4x_find_best_PLL,
 329 };
 330
 331 static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
 332         .dot = { .min = 25000, .max = 350000 },
 333         .vco = { .min = 1760000, .max = 3510000 },
 334         .n = { .min = 1, .max = 3 },
 335         .m = { .min = 79, .max = 126 },
 336         .m1 = { .min = 12, .max = 22 },
 337         .m2 = { .min = 5, .max = 9 },
 338         .p = { .min = 14, .max = 42 },
 339         .p1 = { .min = 2, .max = 6 },
 340         .p2 = { .dot_limit = 225000,
 341                 .p2_slow = 7, .p2_fast = 7 },
 342         .find_pll = intel_g4x_find_best_PLL,
 343 };
 344
 345 static const intel_limit_t intel_limits_ironlake_display_port = {
 346         .dot = { .min = 25000, .max = 350000 },
 347         .vco = { .min = 1760000, .max = 3510000},
 348         .n = { .min = 1, .max = 2 },
 349         .m = { .min = 81, .max = 90 },
 350         .m1 = { .min = 12, .max = 22 },
 351         .m2 = { .min = 5, .max = 9 },
 352         .p = { .min = 10, .max = 20 },
 353         .p1 = { .min = 1, .max = 2},
 354         .p2 = { .dot_limit = 0,
 355                 .p2_slow = 10, .p2_fast = 10 },
 356         .find_pll = intel_find_pll_ironlake_dp,
 357 };
 358
 359 static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
 360                                                 int refclk)
 361 {
 362         struct drm_device *dev = crtc->dev;
 363         struct drm_i915_private *dev_priv = dev->dev_private;
 364         const intel_limit_t *limit;
 365
 366         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
 367                 if ((I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) ==
 368                     LVDS_CLKB_POWER_UP) {
 369                         /* LVDS dual channel */
 370                         if (refclk == 100000)
 371                                 limit = &intel_limits_ironlake_dual_lvds_100m;
 372                         else
 373                                 limit = &intel_limits_ironlake_dual_lvds;
 374                 } else {
 375                         if (refclk == 100000)
 376                                 limit = &intel_limits_ironlake_single_lvds_100m;
 377                         else
 378                                 limit = &intel_limits_ironlake_single_lvds;
 379                 }
 380         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
 381                         HAS_eDP)
 382                 limit = &intel_limits_ironlake_display_port;
 383         else
 384                 limit = &intel_limits_ironlake_dac;
 385
 386         return limit;
 387 }
 388
 389 static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc)
 390 {
 391         struct drm_device *dev = crtc->dev;
 392         struct drm_i915_private *dev_priv = dev->dev_private;
 393         const intel_limit_t *limit;
 394
 395         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
 396                 if ((I915_READ(LVDS) & LVDS_CLKB_POWER_MASK) ==
 397                     LVDS_CLKB_POWER_UP)
 398                         /* LVDS with dual channel */
 399                         limit = &intel_limits_g4x_dual_channel_lvds;
 400                 else
 401                         /* LVDS with dual channel */
 402                         limit = &intel_limits_g4x_single_channel_lvds;
 403         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI) ||
 404                    intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG)) {
 405                 limit = &intel_limits_g4x_hdmi;
 406         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO)) {
 407                 limit = &intel_limits_g4x_sdvo;
 408         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
 409                 limit = &intel_limits_g4x_display_port;
 410         } else /* The option is for other outputs */
 411                 limit = &intel_limits_i9xx_sdvo;
 412
 413         return limit;
 414 }
 415
 416 static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk)
 417 {
 418         struct drm_device *dev = crtc->dev;
 419         const intel_limit_t *limit;
 420
 421         if (HAS_PCH_SPLIT(dev))
 422                 limit = intel_ironlake_limit(crtc, refclk);
 423         else if (IS_G4X(dev)) {
 424                 limit = intel_g4x_limit(crtc);
 425         } else if (IS_PINEVIEW(dev)) {
 426                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
 427                         limit = &intel_limits_pineview_lvds;
 428                 else
 429                         limit = &intel_limits_pineview_sdvo;
 430         } else if (!IS_GEN2(dev)) {
 431                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
 432                         limit = &intel_limits_i9xx_lvds;
 433                 else
 434                         limit = &intel_limits_i9xx_sdvo;
 435         } else {
 436                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
 437                         limit = &intel_limits_i8xx_lvds;
 438                 else
 439                         limit = &intel_limits_i8xx_dvo;
 440         }
 441         return limit;
 442 }
 443
 444 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 445 static void pineview_clock(int refclk, intel_clock_t *clock)
 446 {
 447         clock->m = clock->m2 + 2;
 448         clock->p = clock->p1 * clock->p2;
 449         clock->vco = refclk * clock->m / clock->n;
 450         clock->dot = clock->vco / clock->p;
 451 }
 452
 453 static void intel_clock(struct drm_device *dev, int refclk, intel_clock_t *clock)
 454 {
 455         if (IS_PINEVIEW(dev)) {
 456                 pineview_clock(refclk, clock);
 457                 return;
 458         }
 459         clock->m = 5 * (clock->m1 + 2) + (clock->m2 + 2);
 460         clock->p = clock->p1 * clock->p2;
 461         clock->vco = refclk * clock->m / (clock->n + 2);
 462         clock->dot = clock->vco / clock->p;
 463 }
 464
 465 /**
 466  * Returns whether any output on the specified pipe is of the specified type
 467  */
 468 bool intel_pipe_has_type(struct drm_crtc *crtc, int type)
 469 {
 470         struct drm_device *dev = crtc->dev;
 471         struct drm_mode_config *mode_config = &dev->mode_config;
 472         struct intel_encoder *encoder;
 473
 474         list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
 475                 if (encoder->base.crtc == crtc && encoder->type == type)
 476                         return true;
 477
 478         return false;
 479 }
 480
 481 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 482 /**
 483  * Returns whether the given set of divisors are valid for a given refclk with
 484  * the given connectors.
 485  */
 486
 487 static bool intel_PLL_is_valid(struct drm_device *dev,
 488                                const intel_limit_t *limit,
 489                                const intel_clock_t *clock)
 490 {
 491         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 492                 INTELPllInvalid("p1 out of range\n");
 493         if (clock->p   < limit->p.min   || limit->p.max   < clock->p)
 494                 INTELPllInvalid("p out of range\n");
 495         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 496                 INTELPllInvalid("m2 out of range\n");
 497         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 498                 INTELPllInvalid("m1 out of range\n");
 499         if (clock->m1 <= clock->m2 && !IS_PINEVIEW(dev))
 500                 INTELPllInvalid("m1 <= m2\n");
 501         if (clock->m   < limit->m.min   || limit->m.max   < clock->m)
 502                 INTELPllInvalid("m out of range\n");
 503         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 504                 INTELPllInvalid("n out of range\n");
 505         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 506                 INTELPllInvalid("vco out of range\n");
 507         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 508          * connector, etc., rather than just a single range.
 509          */
 510         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 511                 INTELPllInvalid("dot out of range\n");
 512
 513         return true;
 514 }
 515
 516 static bool
 517 intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
 518                     int target, int refclk, intel_clock_t *best_clock)
 519
 520 {
 521         struct drm_device *dev = crtc->dev;
 522         struct drm_i915_private *dev_priv = dev->dev_private;
 523         intel_clock_t clock;
 524         int err = target;
 525
 526         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
 527             (I915_READ(LVDS)) != 0) {
 528                 /*
 529                  * For LVDS, if the panel is on, just rely on its current
 530                  * settings for dual-channel.  We haven't figured out how to
 531                  * reliably set up different single/dual channel state, if we
 532                  * even can.
 533                  */
 534                 if ((I915_READ(LVDS) & LVDS_CLKB_POWER_MASK) ==
 535                     LVDS_CLKB_POWER_UP)
 536                         clock.p2 = limit->p2.p2_fast;
 537                 else
 538                         clock.p2 = limit->p2.p2_slow;
 539         } else {
 540                 if (target < limit->p2.dot_limit)
 541                         clock.p2 = limit->p2.p2_slow;
 542                 else
 543                         clock.p2 = limit->p2.p2_fast;
 544         }
 545
 546         memset(best_clock, 0, sizeof(*best_clock));
 547
 548         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 549              clock.m1++) {
 550                 for (clock.m2 = limit->m2.min;
 551                      clock.m2 <= limit->m2.max; clock.m2++) {
 552                         /* m1 is always 0 in Pineview */
 553                         if (clock.m2 >= clock.m1 && !IS_PINEVIEW(dev))
 554                                 break;
 555                         for (clock.n = limit->n.min;
 556                              clock.n <= limit->n.max; clock.n++) {
 557                                 for (clock.p1 = limit->p1.min;
 558                                         clock.p1 <= limit->p1.max; clock.p1++) {
 559                                         int this_err;
 560
 561                                         intel_clock(dev, refclk, &clock);
 562                                         if (!intel_PLL_is_valid(dev, limit,
 563                                                                 &clock))
 564                                                 continue;
 565
 566                                         this_err = abs(clock.dot - target);
 567                                         if (this_err < err) {
 568                                                 *best_clock = clock;
 569                                                 err = this_err;
 570                                         }
 571                                 }
 572                         }
 573                 }
 574         }
 575
 576         return (err != target);
 577 }
 578
 579 static bool
 580 intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
 581                         int target, int refclk, intel_clock_t *best_clock)
 582 {
 583         struct drm_device *dev = crtc->dev;
 584         struct drm_i915_private *dev_priv = dev->dev_private;
 585         intel_clock_t clock;
 586         int max_n;
 587         bool found;
 588         /* approximately equals target * 0.00585 */
 589         int err_most = (target >> 8) + (target >> 9);
 590         found = false;
 591
 592         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
 593                 int lvds_reg;
 594
 595                 if (HAS_PCH_SPLIT(dev))
 596                         lvds_reg = PCH_LVDS;
 597                 else
 598                         lvds_reg = LVDS;
 599                 if ((I915_READ(lvds_reg) & LVDS_CLKB_POWER_MASK) ==
 600                     LVDS_CLKB_POWER_UP)
 601                         clock.p2 = limit->p2.p2_fast;
 602                 else
 603                         clock.p2 = limit->p2.p2_slow;
 604         } else {
 605                 if (target < limit->p2.dot_limit)
 606                         clock.p2 = limit->p2.p2_slow;
 607                 else
 608                         clock.p2 = limit->p2.p2_fast;
 609         }
 610
 611         memset(best_clock, 0, sizeof(*best_clock));
 612         max_n = limit->n.max;
 613         /* based on hardware requirement, prefer smaller n to precision */
 614         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 615                 /* based on hardware requirement, prefere larger m1,m2 */
 616                 for (clock.m1 = limit->m1.max;
 617                      clock.m1 >= limit->m1.min; clock.m1--) {
 618                         for (clock.m2 = limit->m2.max;
 619                              clock.m2 >= limit->m2.min; clock.m2--) {
 620                                 for (clock.p1 = limit->p1.max;
 621                                      clock.p1 >= limit->p1.min; clock.p1--) {
 622                                         int this_err;
 623
 624                                         intel_clock(dev, refclk, &clock);
 625                                         if (!intel_PLL_is_valid(dev, limit,
 626                                                                 &clock))
 627                                                 continue;
 628
 629                                         this_err = abs(clock.dot - target);
 630                                         if (this_err < err_most) {
 631                                                 *best_clock = clock;
 632                                                 err_most = this_err;
 633                                                 max_n = clock.n;
 634                                                 found = true;
 635                                         }
 636                                 }
 637                         }
 638                 }
 639         }
 640         return found;
 641 }
 642
 643 static bool
 644 intel_find_pll_ironlake_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
 645                            int target, int refclk, intel_clock_t *best_clock)
 646 {
 647         struct drm_device *dev = crtc->dev;
 648         intel_clock_t clock;
 649
 650         if (target < 200000) {
 651                 clock.n = 1;
 652                 clock.p1 = 2;
 653                 clock.p2 = 10;
 654                 clock.m1 = 12;
 655                 clock.m2 = 9;
 656         } else {
 657                 clock.n = 2;
 658                 clock.p1 = 1;
 659                 clock.p2 = 10;
 660                 clock.m1 = 14;
 661                 clock.m2 = 8;
 662         }
 663         intel_clock(dev, refclk, &clock);
 664         memcpy(best_clock, &clock, sizeof(intel_clock_t));
 665         return true;
 666 }
 667
 668 /* DisplayPort has only two frequencies, 162MHz and 270MHz */
 669 static bool
 670 intel_find_pll_g4x_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
 671                       int target, int refclk, intel_clock_t *best_clock)
 672 {
 673         intel_clock_t clock;
 674         if (target < 200000) {
 675                 clock.p1 = 2;
 676                 clock.p2 = 10;
 677                 clock.n = 2;
 678                 clock.m1 = 23;
 679                 clock.m2 = 8;
 680         } else {
 681                 clock.p1 = 1;
 682                 clock.p2 = 10;
 683                 clock.n = 1;
 684                 clock.m1 = 14;
 685                 clock.m2 = 2;
 686         }
 687         clock.m = 5 * (clock.m1 + 2) + (clock.m2 + 2);
 688         clock.p = (clock.p1 * clock.p2);
 689         clock.dot = 96000 * clock.m / (clock.n + 2) / clock.p;
 690         clock.vco = 0;
 691         memcpy(best_clock, &clock, sizeof(intel_clock_t));
 692         return true;
 693 }
 694
 695 /**
 696  * intel_wait_for_vblank - wait for vblank on a given pipe
 697  * @dev: drm device
 698  * @pipe: pipe to wait for
 699  *
 700  * Wait for vblank to occur on a given pipe.  Needed for various bits of
 701  * mode setting code.
 702  */
 703 void intel_wait_for_vblank(struct drm_device *dev, int pipe)
 704 {
 705         struct drm_i915_private *dev_priv = dev->dev_private;
 706         int pipestat_reg = PIPESTAT(pipe);
 707
 708         /* Clear existing vblank status. Note this will clear any other
 709          * sticky status fields as well.
 710          *
 711          * This races with i915_driver_irq_handler() with the result
 712          * that either function could miss a vblank event.  Here it is not
 713          * fatal, as we will either wait upon the next vblank interrupt or
 714          * timeout.  Generally speaking intel_wait_for_vblank() is only
 715          * called during modeset at which time the GPU should be idle and
 716          * should *not* be performing page flips and thus not waiting on
 717          * vblanks...
 718          * Currently, the result of us stealing a vblank from the irq
 719          * handler is that a single frame will be skipped during swapbuffers.
 720          */
 721         I915_WRITE(pipestat_reg,
 722                    I915_READ(pipestat_reg) | PIPE_VBLANK_INTERRUPT_STATUS);
 723
 724         /* Wait for vblank interrupt bit to set */
 725         if (wait_for(I915_READ(pipestat_reg) &
 726                      PIPE_VBLANK_INTERRUPT_STATUS,
 727                      50))
 728                 DRM_DEBUG_KMS("vblank wait timed out\n");
 729 }
 730
 731 /*
 732  * intel_wait_for_pipe_off - wait for pipe to turn off
 733  * @dev: drm device
 734  * @pipe: pipe to wait for
 735  *
 736  * After disabling a pipe, we can't wait for vblank in the usual way,
 737  * spinning on the vblank interrupt status bit, since we won't actually
 738  * see an interrupt when the pipe is disabled.
 739  *
 740  * On Gen4 and above:
 741  *   wait for the pipe register state bit to turn off
 742  *
 743  * Otherwise:
 744  *   wait for the display line value to settle (it usually
 745  *   ends up stopping at the start of the next frame).
 746  *
 747  */
 748 void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
 749 {
 750         struct drm_i915_private *dev_priv = dev->dev_private;
 751
 752         if (INTEL_INFO(dev)->gen >= 4) {
 753                 int reg = PIPECONF(pipe);
 754
 755                 /* Wait for the Pipe State to go off */
 756                 if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
 757                              100))
 758                         DRM_DEBUG_KMS("pipe_off wait timed out\n");
 759         } else {
 760                 u32 last_line;
 761                 int reg = PIPEDSL(pipe);
 762                 unsigned long timeout = jiffies + msecs_to_jiffies(100);
 763
 764                 /* Wait for the display line to settle */
 765                 do {
 766                         last_line = I915_READ(reg) & DSL_LINEMASK;
 767                         mdelay(5);
 768                 } while (((I915_READ(reg) & DSL_LINEMASK) != last_line) &&
 769                          time_after(timeout, jiffies));
 770                 if (time_after(jiffies, timeout))
 771                         DRM_DEBUG_KMS("pipe_off wait timed out\n");
 772         }
 773 }
 774
 775 static const char *state_string(bool enabled)
 776 {
 777         return enabled ? "on" : "off";
 778 }
 779
 780 /* Only for pre-ILK configs */
 781 static void assert_pll(struct drm_i915_private *dev_priv,
 782                        enum pipe pipe, bool state)
 783 {
 784         int reg;
 785         u32 val;
 786         bool cur_state;
 787
 788         reg = DPLL(pipe);
 789         val = I915_READ(reg);
 790         cur_state = !!(val & DPLL_VCO_ENABLE);
 791         WARN(cur_state != state,
 792              "PLL state assertion failure (expected %s, current %s)\n",
 793              state_string(state), state_string(cur_state));
 794 }
 795 #define assert_pll_enabled(d, p) assert_pll(d, p, true)
 796 #define assert_pll_disabled(d, p) assert_pll(d, p, false)
 797
 798 /* For ILK+ */
 799 static void assert_pch_pll(struct drm_i915_private *dev_priv,
 800                            enum pipe pipe, bool state)
 801 {
 802         int reg;
 803         u32 val;
 804         bool cur_state;
 805
 806         reg = PCH_DPLL(pipe);
 807         val = I915_READ(reg);
 808         cur_state = !!(val & DPLL_VCO_ENABLE);
 809         WARN(cur_state != state,
 810              "PCH PLL state assertion failure (expected %s, current %s)\n",
 811              state_string(state), state_string(cur_state));
 812 }
 813 #define assert_pch_pll_enabled(d, p) assert_pch_pll(d, p, true)
 814 #define assert_pch_pll_disabled(d, p) assert_pch_pll(d, p, false)
 815
 816 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
 817                           enum pipe pipe, bool state)
 818 {
 819         int reg;
 820         u32 val;
 821         bool cur_state;
 822
 823         reg = FDI_TX_CTL(pipe);
 824         val = I915_READ(reg);
 825         cur_state = !!(val & FDI_TX_ENABLE);
 826         WARN(cur_state != state,
 827              "FDI TX state assertion failure (expected %s, current %s)\n",
 828              state_string(state), state_string(cur_state));
 829 }
 830 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
 831 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
 832
 833 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
 834                           enum pipe pipe, bool state)
 835 {
 836         int reg;
 837         u32 val;
 838         bool cur_state;
 839
 840         reg = FDI_RX_CTL(pipe);
 841         val = I915_READ(reg);
 842         cur_state = !!(val & FDI_RX_ENABLE);
 843         WARN(cur_state != state,
 844              "FDI RX state assertion failure (expected %s, current %s)\n",
 845              state_string(state), state_string(cur_state));
 846 }
 847 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
 848 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
 849
 850 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
 851                                       enum pipe pipe)
 852 {
 853         int reg;
 854         u32 val;
 855
 856         /* ILK FDI PLL is always enabled */
 857         if (dev_priv->info->gen == 5)
 858                 return;
 859
 860         reg = FDI_TX_CTL(pipe);
 861         val = I915_READ(reg);
 862         WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
 863 }
 864
 865 static void assert_fdi_rx_pll_enabled(struct drm_i915_private *dev_priv,
 866                                       enum pipe pipe)
 867 {
 868         int reg;
 869         u32 val;
 870
 871         reg = FDI_RX_CTL(pipe);
 872         val = I915_READ(reg);
 873         WARN(!(val & FDI_RX_PLL_ENABLE), "FDI RX PLL assertion failure, should be active but is disabled\n");
 874 }
 875
 876 static void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 877                                   enum pipe pipe)
 878 {
 879         int pp_reg, lvds_reg;
 880         u32 val;
 881         enum pipe panel_pipe = PIPE_A;
 882         bool locked = true;
 883
 884         if (HAS_PCH_SPLIT(dev_priv->dev)) {
 885                 pp_reg = PCH_PP_CONTROL;
 886                 lvds_reg = PCH_LVDS;
 887         } else {
 888                 pp_reg = PP_CONTROL;
 889                 lvds_reg = LVDS;
 890         }
 891
 892         val = I915_READ(pp_reg);
 893         if (!(val & PANEL_POWER_ON) ||
 894             ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS))
 895                 locked = false;
 896
 897         if (I915_READ(lvds_reg) & LVDS_PIPEB_SELECT)
 898                 panel_pipe = PIPE_B;
 899
 900         WARN(panel_pipe == pipe && locked,
 901              "panel assertion failure, pipe %c regs locked\n",
 902              pipe_name(pipe));
 903 }
 904
 905 static void assert_pipe(struct drm_i915_private *dev_priv,
 906                         enum pipe pipe, bool state)
 907 {
 908         int reg;
 909         u32 val;
 910         bool cur_state;
 911
 912         reg = PIPECONF(pipe);
 913         val = I915_READ(reg);
 914         cur_state = !!(val & PIPECONF_ENABLE);
 915         WARN(cur_state != state,
 916              "pipe %c assertion failure (expected %s, current %s)\n",
 917              pipe_name(pipe), state_string(state), state_string(cur_state));
 918 }
 919 #define assert_pipe_enabled(d, p) assert_pipe(d, p, true)
 920 #define assert_pipe_disabled(d, p) assert_pipe(d, p, false)
 921
 922 static void assert_plane_enabled(struct drm_i915_private *dev_priv,
 923                                  enum plane plane)
 924 {
 925         int reg;
 926         u32 val;
 927
 928         reg = DSPCNTR(plane);
 929         val = I915_READ(reg);
 930         WARN(!(val & DISPLAY_PLANE_ENABLE),
 931              "plane %c assertion failure, should be active but is disabled\n",
 932              plane_name(plane));
 933 }
 934
 935 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
 936                                    enum pipe pipe)
 937 {
 938         int reg, i;
 939         u32 val;
 940         int cur_pipe;
 941
 942         /* Planes are fixed to pipes on ILK+ */
 943         if (HAS_PCH_SPLIT(dev_priv->dev))
 944                 return;
 945
 946         /* Need to check both planes against the pipe */
 947         for (i = 0; i < 2; i++) {
 948                 reg = DSPCNTR(i);
 949                 val = I915_READ(reg);
 950                 cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
 951                         DISPPLANE_SEL_PIPE_SHIFT;
 952                 WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe,
 953                      "plane %c assertion failure, should be off on pipe %c but is still active\n",
 954                      plane_name(i), pipe_name(pipe));
 955         }
 956 }
 957
 958 static void assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
 959 {
 960         u32 val;
 961         bool enabled;
 962
 963         val = I915_READ(PCH_DREF_CONTROL);
 964         enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK |
 965                             DREF_SUPERSPREAD_SOURCE_MASK));
 966         WARN(!enabled, "PCH refclk assertion failure, should be active but is disabled\n");
 967 }
 968
 969 static void assert_transcoder_disabled(struct drm_i915_private *dev_priv,
 970                                        enum pipe pipe)
 971 {
 972         int reg;
 973         u32 val;
 974         bool enabled;
 975
 976         reg = TRANSCONF(pipe);
 977         val = I915_READ(reg);
 978         enabled = !!(val & TRANS_ENABLE);
 979         WARN(enabled,
 980              "transcoder assertion failed, should be off on pipe %c but is still active\n",
 981              pipe_name(pipe));
 982 }
 983
 984 static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
 985                             enum pipe pipe, u32 port_sel, u32 val)
 986 {
 987         if ((val & DP_PORT_EN) == 0)
 988                 return false;
 989
 990         if (HAS_PCH_CPT(dev_priv->dev)) {
 991                 u32     trans_dp_ctl_reg = TRANS_DP_CTL(pipe);
 992                 u32     trans_dp_ctl = I915_READ(trans_dp_ctl_reg);
 993                 if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
 994                         return false;
 995         } else {
 996                 if ((val & DP_PIPE_MASK) != (pipe << 30))
 997                         return false;
 998         }
 999         return true;
1000 }
1001
1002 static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv,
1003                               enum pipe pipe, u32 val)
1004 {
1005         if ((val & PORT_ENABLE) == 0)
1006                 return false;
1007
1008         if (HAS_PCH_CPT(dev_priv->dev)) {
1009                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1010                         return false;
1011         } else {
1012                 if ((val & TRANSCODER_MASK) != TRANSCODER(pipe))
1013                         return false;
1014         }
1015         return true;
1016 }
1017
1018 static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv,
1019                               enum pipe pipe, u32 val)
1020 {
1021         if ((val & LVDS_PORT_EN) == 0)
1022                 return false;
1023
1024         if (HAS_PCH_CPT(dev_priv->dev)) {
1025                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1026                         return false;
1027         } else {
1028                 if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe))
1029                         return false;
1030         }
1031         return true;
1032 }
1033
1034 static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
1035                               enum pipe pipe, u32 val)
1036 {
1037         if ((val & ADPA_DAC_ENABLE) == 0)
1038                 return false;
1039         if (HAS_PCH_CPT(dev_priv->dev)) {
1040                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1041                         return false;
1042         } else {
1043                 if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe))
1044                         return false;
1045         }
1046         return true;
1047 }
1048
1049 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1050                                    enum pipe pipe, int reg, u32 port_sel)
1051 {
1052         u32 val = I915_READ(reg);
1053         WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val),
1054              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
1055              reg, pipe_name(pipe));
1056 }
1057
1058 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1059                                      enum pipe pipe, int reg)
1060 {
1061         u32 val = I915_READ(reg);
1062         WARN(hdmi_pipe_enabled(dev_priv, val, pipe),
1063              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
1064              reg, pipe_name(pipe));
1065 }
1066
1067 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1068                                       enum pipe pipe)
1069 {
1070         int reg;
1071         u32 val;
1072
1073         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
1074         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
1075         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
1076
1077         reg = PCH_ADPA;
1078         val = I915_READ(reg);
1079         WARN(adpa_pipe_enabled(dev_priv, val, pipe),
1080              "PCH VGA enabled on transcoder %c, should be disabled\n",
1081              pipe_name(pipe));
1082
1083         reg = PCH_LVDS;
1084         val = I915_READ(reg);
1085         WARN(lvds_pipe_enabled(dev_priv, val, pipe),
1086              "PCH LVDS enabled on transcoder %c, should be disabled\n",
1087              pipe_name(pipe));
1088
1089         assert_pch_hdmi_disabled(dev_priv, pipe, HDMIB);
1090         assert_pch_hdmi_disabled(dev_priv, pipe, HDMIC);
1091         assert_pch_hdmi_disabled(dev_priv, pipe, HDMID);
1092 }
1093
1094 /**
1095  * intel_enable_pll - enable a PLL
1096  * @dev_priv: i915 private structure
1097  * @pipe: pipe PLL to enable
1098  *
1099  * Enable @pipe's PLL so we can start pumping pixels from a plane.  Check to
1100  * make sure the PLL reg is writable first though, since the panel write
1101  * protect mechanism may be enabled.
1102  *
1103  * Note!  This is for pre-ILK only.
1104  */
1105 static void intel_enable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1106 {
1107         int reg;
1108         u32 val;
1109
1110         /* No really, not for ILK+ */
1111         BUG_ON(dev_priv->info->gen >= 5);
1112
1113         /* PLL is protected by panel, make sure we can write it */
1114         if (IS_MOBILE(dev_priv->dev) && !IS_I830(dev_priv->dev))
1115                 assert_panel_unlocked(dev_priv, pipe);
1116
1117         reg = DPLL(pipe);
1118         val = I915_READ(reg);
1119         val |= DPLL_VCO_ENABLE;
1120
1121         /* We do this three times for luck */
1122         I915_WRITE(reg, val);
1123         POSTING_READ(reg);
1124         udelay(150); /* wait for warmup */
1125         I915_WRITE(reg, val);
1126         POSTING_READ(reg);
1127         udelay(150); /* wait for warmup */
1128         I915_WRITE(reg, val);
1129         POSTING_READ(reg);
1130         udelay(150); /* wait for warmup */
1131 }
1132
1133 /**
1134  * intel_disable_pll - disable a PLL
1135  * @dev_priv: i915 private structure
1136  * @pipe: pipe PLL to disable
1137  *
1138  * Disable the PLL for @pipe, making sure the pipe is off first.
1139  *
1140  * Note!  This is for pre-ILK only.
1141  */
1142 static void intel_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1143 {
1144         int reg;
1145         u32 val;
1146
1147         /* Don't disable pipe A or pipe A PLLs if needed */
1148         if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
1149                 return;
1150
1151         /* Make sure the pipe isn't still relying on us */
1152         assert_pipe_disabled(dev_priv, pipe);
1153
1154         reg = DPLL(pipe);
1155         val = I915_READ(reg);
1156         val &= ~DPLL_VCO_ENABLE;
1157         I915_WRITE(reg, val);
1158         POSTING_READ(reg);
1159 }
1160
1161 /**
1162  * intel_enable_pch_pll - enable PCH PLL
1163  * @dev_priv: i915 private structure
1164  * @pipe: pipe PLL to enable
1165  *
1166  * The PCH PLL needs to be enabled before the PCH transcoder, since it
1167  * drives the transcoder clock.
1168  */
1169 static void intel_enable_pch_pll(struct drm_i915_private *dev_priv,
1170                                  enum pipe pipe)
1171 {
1172         int reg;
1173         u32 val;
1174
1175         if (pipe > 1)
1176                 return;
1177
1178         /* PCH only available on ILK+ */
1179         BUG_ON(dev_priv->info->gen < 5);
1180
1181         /* PCH refclock must be enabled first */
1182         assert_pch_refclk_enabled(dev_priv);
1183
1184         reg = PCH_DPLL(pipe);
1185         val = I915_READ(reg);
1186         val |= DPLL_VCO_ENABLE;
1187         I915_WRITE(reg, val);
1188         POSTING_READ(reg);
1189         udelay(200);
1190 }
1191
1192 static void intel_disable_pch_pll(struct drm_i915_private *dev_priv,
1193                                   enum pipe pipe)
1194 {
1195         int reg;
1196         u32 val;
1197
1198         if (pipe > 1)
1199                 return;
1200
1201         /* PCH only available on ILK+ */
1202         BUG_ON(dev_priv->info->gen < 5);
1203
1204         /* Make sure transcoder isn't still depending on us */
1205         assert_transcoder_disabled(dev_priv, pipe);
1206
1207         reg = PCH_DPLL(pipe);
1208         val = I915_READ(reg);
1209         val &= ~DPLL_VCO_ENABLE;
1210         I915_WRITE(reg, val);
1211         POSTING_READ(reg);
1212         udelay(200);
1213 }
1214
1215 static void intel_enable_transcoder(struct drm_i915_private *dev_priv,
1216                                     enum pipe pipe)
1217 {
1218         int reg;
1219         u32 val;
1220
1221         /* PCH only available on ILK+ */
1222         BUG_ON(dev_priv->info->gen < 5);
1223
1224         /* Make sure PCH DPLL is enabled */
1225         assert_pch_pll_enabled(dev_priv, pipe);
1226
1227         /* FDI must be feeding us bits for PCH ports */
1228         assert_fdi_tx_enabled(dev_priv, pipe);
1229         assert_fdi_rx_enabled(dev_priv, pipe);
1230
1231         reg = TRANSCONF(pipe);
1232         val = I915_READ(reg);
1233
1234         if (HAS_PCH_IBX(dev_priv->dev)) {
1235                 /*
1236                  * make the BPC in transcoder be consistent with
1237                  * that in pipeconf reg.
1238                  */
1239                 val &= ~PIPE_BPC_MASK;
1240                 val |= I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK;
1241         }
1242         I915_WRITE(reg, val | TRANS_ENABLE);
1243         if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
1244                 DRM_ERROR("failed to enable transcoder %d\n", pipe);
1245 }
1246
1247 static void intel_disable_transcoder(struct drm_i915_private *dev_priv,
1248                                      enum pipe pipe)
1249 {
1250         int reg;
1251         u32 val;
1252
1253         /* FDI relies on the transcoder */
1254         assert_fdi_tx_disabled(dev_priv, pipe);
1255         assert_fdi_rx_disabled(dev_priv, pipe);
1256
1257         /* Ports must be off as well */
1258         assert_pch_ports_disabled(dev_priv, pipe);
1259
1260         reg = TRANSCONF(pipe);
1261         val = I915_READ(reg);
1262         val &= ~TRANS_ENABLE;
1263         I915_WRITE(reg, val);
1264         /* wait for PCH transcoder off, transcoder state */
1265         if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
1266                 DRM_ERROR("failed to disable transcoder\n");
1267 }
1268
1269 /**
1270  * intel_enable_pipe - enable a pipe, asserting requirements
1271  * @dev_priv: i915 private structure
1272  * @pipe: pipe to enable
1273  * @pch_port: on ILK+, is this pipe driving a PCH port or not
1274  *
1275  * Enable @pipe, making sure that various hardware specific requirements
1276  * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
1277  *
1278  * @pipe should be %PIPE_A or %PIPE_B.
1279  *
1280  * Will wait until the pipe is actually running (i.e. first vblank) before
1281  * returning.
1282  */
1283 static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
1284                               bool pch_port)
1285 {
1286         int reg;
1287         u32 val;
1288
1289         /*
1290          * A pipe without a PLL won't actually be able to drive bits from
1291          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1292          * need the check.
1293          */
1294         if (!HAS_PCH_SPLIT(dev_priv->dev))
1295                 assert_pll_enabled(dev_priv, pipe);
1296         else {
1297                 if (pch_port) {
1298                         /* if driving the PCH, we need FDI enabled */
1299                         assert_fdi_rx_pll_enabled(dev_priv, pipe);
1300                         assert_fdi_tx_pll_enabled(dev_priv, pipe);
1301                 }
1302                 /* FIXME: assert CPU port conditions for SNB+ */
1303         }
1304
1305         reg = PIPECONF(pipe);
1306         val = I915_READ(reg);
1307         if (val & PIPECONF_ENABLE)
1308                 return;
1309
1310         I915_WRITE(reg, val | PIPECONF_ENABLE);
1311         intel_wait_for_vblank(dev_priv->dev, pipe);
1312 }
1313
1314 /**
1315  * intel_disable_pipe - disable a pipe, asserting requirements
1316  * @dev_priv: i915 private structure
1317  * @pipe: pipe to disable
1318  *
1319  * Disable @pipe, making sure that various hardware specific requirements
1320  * are met, if applicable, e.g. plane disabled, panel fitter off, etc.
1321  *
1322  * @pipe should be %PIPE_A or %PIPE_B.
1323  *
1324  * Will wait until the pipe has shut down before returning.
1325  */
1326 static void intel_disable_pipe(struct drm_i915_private *dev_priv,
1327                                enum pipe pipe)
1328 {
1329         int reg;
1330         u32 val;
1331
1332         /*
1333          * Make sure planes won't keep trying to pump pixels to us,
1334          * or we might hang the display.
1335          */
1336         assert_planes_disabled(dev_priv, pipe);
1337
1338         /* Don't disable pipe A or pipe A PLLs if needed */
1339         if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
1340                 return;
1341
1342         reg = PIPECONF(pipe);
1343         val = I915_READ(reg);
1344         if ((val & PIPECONF_ENABLE) == 0)
1345                 return;
1346
1347         I915_WRITE(reg, val & ~PIPECONF_ENABLE);
1348         intel_wait_for_pipe_off(dev_priv->dev, pipe);
1349 }
1350
1351 /*
1352  * Plane regs are double buffered, going from enabled->disabled needs a
1353  * trigger in order to latch.  The display address reg provides this.
1354  */
1355 static void intel_flush_display_plane(struct drm_i915_private *dev_priv,
1356                                       enum plane plane)
1357 {
1358         I915_WRITE(DSPADDR(plane), I915_READ(DSPADDR(plane)));
1359         I915_WRITE(DSPSURF(plane), I915_READ(DSPSURF(plane)));
1360 }
1361
1362 /**
1363  * intel_enable_plane - enable a display plane on a given pipe
1364  * @dev_priv: i915 private structure
1365  * @plane: plane to enable
1366  * @pipe: pipe being fed
1367  *
1368  * Enable @plane on @pipe, making sure that @pipe is running first.
1369  */
1370 static void intel_enable_plane(struct drm_i915_private *dev_priv,
1371                                enum plane plane, enum pipe pipe)
1372 {
1373         int reg;
1374         u32 val;
1375
1376         /* If the pipe isn't enabled, we can't pump pixels and may hang */
1377         assert_pipe_enabled(dev_priv, pipe);
1378
1379         reg = DSPCNTR(plane);
1380         val = I915_READ(reg);
1381         if (val & DISPLAY_PLANE_ENABLE)
1382                 return;
1383
1384         I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE);
1385         intel_flush_display_plane(dev_priv, plane);
1386         intel_wait_for_vblank(dev_priv->dev, pipe);
1387 }
1388
1389 /**
1390  * intel_disable_plane - disable a display plane
1391  * @dev_priv: i915 private structure
1392  * @plane: plane to disable
1393  * @pipe: pipe consuming the data
1394  *
1395  * Disable @plane; should be an independent operation.
1396  */
1397 static void intel_disable_plane(struct drm_i915_private *dev_priv,
1398                                 enum plane plane, enum pipe pipe)
1399 {
1400         int reg;
1401         u32 val;
1402
1403         reg = DSPCNTR(plane);
1404         val = I915_READ(reg);
1405         if ((val & DISPLAY_PLANE_ENABLE) == 0)
1406                 return;
1407
1408         I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE);
1409         intel_flush_display_plane(dev_priv, plane);
1410         intel_wait_for_vblank(dev_priv->dev, pipe);
1411 }
1412
1413 static void disable_pch_dp(struct drm_i915_private *dev_priv,
1414                            enum pipe pipe, int reg, u32 port_sel)
1415 {
1416         u32 val = I915_READ(reg);
1417         if (dp_pipe_enabled(dev_priv, pipe, port_sel, val)) {
1418                 DRM_DEBUG_KMS("Disabling pch dp %x on pipe %d\n", reg, pipe);
1419                 I915_WRITE(reg, val & ~DP_PORT_EN);
1420         }
1421 }
1422
1423 static void disable_pch_hdmi(struct drm_i915_private *dev_priv,
1424                              enum pipe pipe, int reg)
1425 {
1426         u32 val = I915_READ(reg);
1427         if (hdmi_pipe_enabled(dev_priv, val, pipe)) {
1428                 DRM_DEBUG_KMS("Disabling pch HDMI %x on pipe %d\n",
1429                               reg, pipe);
1430                 I915_WRITE(reg, val & ~PORT_ENABLE);
1431         }
1432 }
1433
1434 /* Disable any ports connected to this transcoder */
1435 static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
1436                                     enum pipe pipe)
1437 {
1438         u32 reg, val;
1439
1440         val = I915_READ(PCH_PP_CONTROL);
1441         I915_WRITE(PCH_PP_CONTROL, val | PANEL_UNLOCK_REGS);
1442
1443         disable_pch_dp(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
1444         disable_pch_dp(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
1445         disable_pch_dp(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
1446
1447         reg = PCH_ADPA;
1448         val = I915_READ(reg);
1449         if (adpa_pipe_enabled(dev_priv, val, pipe))
1450                 I915_WRITE(reg, val & ~ADPA_DAC_ENABLE);
1451
1452         reg = PCH_LVDS;
1453         val = I915_READ(reg);
1454         if (lvds_pipe_enabled(dev_priv, val, pipe)) {
1455                 DRM_DEBUG_KMS("disable lvds on pipe %d val 0x%08x\n", pipe, val);
1456                 I915_WRITE(reg, val & ~LVDS_PORT_EN);
1457                 POSTING_READ(reg);
1458                 udelay(100);
1459         }
1460
1461         disable_pch_hdmi(dev_priv, pipe, HDMIB);
1462         disable_pch_hdmi(dev_priv, pipe, HDMIC);
1463         disable_pch_hdmi(dev_priv, pipe, HDMID);
1464 }
1465
1466 static void i8xx_disable_fbc(struct drm_device *dev)
1467 {
1468         struct drm_i915_private *dev_priv = dev->dev_private;
1469         u32 fbc_ctl;
1470
1471         /* Disable compression */
1472         fbc_ctl = I915_READ(FBC_CONTROL);
1473         if ((fbc_ctl & FBC_CTL_EN) == 0)
1474                 return;
1475
1476         fbc_ctl &= ~FBC_CTL_EN;
1477         I915_WRITE(FBC_CONTROL, fbc_ctl);
1478
1479         /* Wait for compressing bit to clear */
1480         if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
1481                 DRM_DEBUG_KMS("FBC idle timed out\n");
1482                 return;
1483         }
1484
1485         DRM_DEBUG_KMS("disabled FBC\n");
1486 }
1487
1488 static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1489 {
1490         struct drm_device *dev = crtc->dev;
1491         struct drm_i915_private *dev_priv = dev->dev_private;
1492         struct drm_framebuffer *fb = crtc->fb;
1493         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1494         struct drm_i915_gem_object *obj = intel_fb->obj;
1495         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1496         int cfb_pitch;
1497         int plane, i;
1498         u32 fbc_ctl, fbc_ctl2;
1499
1500         cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
1501         if (fb->pitch < cfb_pitch)
1502                 cfb_pitch = fb->pitch;
1503
1504         /* FBC_CTL wants 64B units */
1505         cfb_pitch = (cfb_pitch / 64) - 1;
1506         plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
1507
1508         /* Clear old tags */
1509         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
1510                 I915_WRITE(FBC_TAG + (i * 4), 0);
1511
1512         /* Set it up... */
1513         fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
1514         fbc_ctl2 |= plane;
1515         I915_WRITE(FBC_CONTROL2, fbc_ctl2);
1516         I915_WRITE(FBC_FENCE_OFF, crtc->y);
1517
1518         /* enable it... */
1519         fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
1520         if (IS_I945GM(dev))
1521                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
1522         fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
1523         fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
1524         fbc_ctl |= obj->fence_reg;
1525         I915_WRITE(FBC_CONTROL, fbc_ctl);
1526
1527         DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
1528                       cfb_pitch, crtc->y, intel_crtc->plane);
1529 }
1530
1531 static bool i8xx_fbc_enabled(struct drm_device *dev)
1532 {
1533         struct drm_i915_private *dev_priv = dev->dev_private;
1534
1535         return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
1536 }
1537
1538 static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1539 {
1540         struct drm_device *dev = crtc->dev;
1541         struct drm_i915_private *dev_priv = dev->dev_private;
1542         struct drm_framebuffer *fb = crtc->fb;
1543         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1544         struct drm_i915_gem_object *obj = intel_fb->obj;
1545         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1546         int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
1547         unsigned long stall_watermark = 200;
1548         u32 dpfc_ctl;
1549
1550         dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
1551         dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
1552         I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
1553
1554         I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
1555                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
1556                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
1557         I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
1558
1559         /* enable it... */
1560         I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
1561
1562         DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
1563 }
1564
1565 static void g4x_disable_fbc(struct drm_device *dev)
1566 {
1567         struct drm_i915_private *dev_priv = dev->dev_private;
1568         u32 dpfc_ctl;
1569
1570         /* Disable compression */
1571         dpfc_ctl = I915_READ(DPFC_CONTROL);
1572         if (dpfc_ctl & DPFC_CTL_EN) {
1573                 dpfc_ctl &= ~DPFC_CTL_EN;
1574                 I915_WRITE(DPFC_CONTROL, dpfc_ctl);
1575
1576                 DRM_DEBUG_KMS("disabled FBC\n");
1577         }
1578 }
1579
1580 static bool g4x_fbc_enabled(struct drm_device *dev)
1581 {
1582         struct drm_i915_private *dev_priv = dev->dev_private;
1583
1584         return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
1585 }
1586
1587 static void sandybridge_blit_fbc_update(struct drm_device *dev)
1588 {
1589         struct drm_i915_private *dev_priv = dev->dev_private;
1590         u32 blt_ecoskpd;
1591
1592         /* Make sure blitter notifies FBC of writes */
1593         gen6_gt_force_wake_get(dev_priv);
1594         blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
1595         blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
1596                 GEN6_BLITTER_LOCK_SHIFT;
1597         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
1598         blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
1599         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
1600         blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
1601                          GEN6_BLITTER_LOCK_SHIFT);
1602         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
1603         POSTING_READ(GEN6_BLITTER_ECOSKPD);
1604         gen6_gt_force_wake_put(dev_priv);
1605 }
1606
1607 static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1608 {
1609         struct drm_device *dev = crtc->dev;
1610         struct drm_i915_private *dev_priv = dev->dev_private;
1611         struct drm_framebuffer *fb = crtc->fb;
1612         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1613         struct drm_i915_gem_object *obj = intel_fb->obj;
1614         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1615         int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
1616         unsigned long stall_watermark = 200;
1617         u32 dpfc_ctl;
1618
1619         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
1620         dpfc_ctl &= DPFC_RESERVED;
1621         dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
1622         /* Set persistent mode for front-buffer rendering, ala X. */
1623         dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
1624         dpfc_ctl |= (DPFC_CTL_FENCE_EN | obj->fence_reg);
1625         I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
1626
1627         I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
1628                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
1629                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
1630         I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
1631         I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
1632         /* enable it... */
1633         I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
1634
1635         if (IS_GEN6(dev)) {
1636                 I915_WRITE(SNB_DPFC_CTL_SA,
1637                            SNB_CPU_FENCE_ENABLE | obj->fence_reg);
1638                 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
1639                 sandybridge_blit_fbc_update(dev);
1640         }
1641
1642         DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
1643 }
1644
1645 static void ironlake_disable_fbc(struct drm_device *dev)
1646 {
1647         struct drm_i915_private *dev_priv = dev->dev_private;
1648         u32 dpfc_ctl;
1649
1650         /* Disable compression */
1651         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
1652         if (dpfc_ctl & DPFC_CTL_EN) {
1653                 dpfc_ctl &= ~DPFC_CTL_EN;
1654                 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
1655
1656                 DRM_DEBUG_KMS("disabled FBC\n");
1657         }
1658 }
1659
1660 static bool ironlake_fbc_enabled(struct drm_device *dev)
1661 {
1662         struct drm_i915_private *dev_priv = dev->dev_private;
1663
1664         return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
1665 }
1666
1667 bool intel_fbc_enabled(struct drm_device *dev)
1668 {
1669         struct drm_i915_private *dev_priv = dev->dev_private;
1670
1671         if (!dev_priv->display.fbc_enabled)
1672                 return false;
1673
1674         return dev_priv->display.fbc_enabled(dev);
1675 }
1676
1677 static void intel_fbc_work_fn(struct work_struct *__work)
1678 {
1679         struct intel_fbc_work *work =
1680                 container_of(to_delayed_work(__work),
1681                              struct intel_fbc_work, work);
1682         struct drm_device *dev = work->crtc->dev;
1683         struct drm_i915_private *dev_priv = dev->dev_private;
1684
1685         mutex_lock(&dev->struct_mutex);
1686         if (work == dev_priv->fbc_work) {
1687                 /* Double check that we haven't switched fb without cancelling
1688                  * the prior work.
1689                  */
1690                 if (work->crtc->fb == work->fb) {
1691                         dev_priv->display.enable_fbc(work->crtc,
1692                                                      work->interval);
1693
1694                         dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
1695                         dev_priv->cfb_fb = work->crtc->fb->base.id;
1696                         dev_priv->cfb_y = work->crtc->y;
1697                 }
1698
1699                 dev_priv->fbc_work = NULL;
1700         }
1701         mutex_unlock(&dev->struct_mutex);
1702
1703         kfree(work);
1704 }
1705
1706 static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
1707 {
1708         if (dev_priv->fbc_work == NULL)
1709                 return;
1710
1711         DRM_DEBUG_KMS("cancelling pending FBC enable\n");
1712
1713         /* Synchronisation is provided by struct_mutex and checking of
1714          * dev_priv->fbc_work, so we can perform the cancellation
1715          * entirely asynchronously.
1716          */
1717         if (cancel_delayed_work(&dev_priv->fbc_work->work))
1718                 /* tasklet was killed before being run, clean up */
1719                 kfree(dev_priv->fbc_work);
1720
1721         /* Mark the work as no longer wanted so that if it does
1722          * wake-up (because the work was already running and waiting
1723          * for our mutex), it will discover that is no longer
1724          * necessary to run.
1725          */
1726         dev_priv->fbc_work = NULL;
1727 }
1728
1729 static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1730 {
1731         struct intel_fbc_work *work;
1732         struct drm_device *dev = crtc->dev;
1733         struct drm_i915_private *dev_priv = dev->dev_private;
1734
1735         if (!dev_priv->display.enable_fbc)
1736                 return;
1737
1738         intel_cancel_fbc_work(dev_priv);
1739
1740         work = kzalloc(sizeof *work, GFP_KERNEL);
1741         if (work == NULL) {
1742                 dev_priv->display.enable_fbc(crtc, interval);
1743                 return;
1744         }
1745
1746         work->crtc = crtc;
1747         work->fb = crtc->fb;
1748         work->interval = interval;
1749         INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
1750
1751         dev_priv->fbc_work = work;
1752
1753         DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
1754
1755         /* Delay the actual enabling to let pageflipping cease and the
1756          * display to settle before starting the compression. Note that
1757          * this delay also serves a second purpose: it allows for a
1758          * vblank to pass after disabling the FBC before we attempt
1759          * to modify the control registers.
1760          *
1761          * A more complicated solution would involve tracking vblanks
1762          * following the termination of the page-flipping sequence
1763          * and indeed performing the enable as a co-routine and not
1764          * waiting synchronously upon the vblank.
1765          */
1766         schedule_delayed_work(&work->work, msecs_to_jiffies(50));
1767 }
1768
1769 void intel_disable_fbc(struct drm_device *dev)
1770 {
1771         struct drm_i915_private *dev_priv = dev->dev_private;
1772
1773         intel_cancel_fbc_work(dev_priv);
1774
1775         if (!dev_priv->display.disable_fbc)
1776                 return;
1777
1778         dev_priv->display.disable_fbc(dev);
1779         dev_priv->cfb_plane = -1;
1780 }
1781
1782 /**
1783  * intel_update_fbc - enable/disable FBC as needed
1784  * @dev: the drm_device
1785  *
1786  * Set up the framebuffer compression hardware at mode set time.  We
1787  * enable it if possible:
1788  *   - plane A only (on pre-965)
1789  *   - no pixel mulitply/line duplication
1790  *   - no alpha buffer discard
1791  *   - no dual wide
1792  *   - framebuffer <= 2048 in width, 1536 in height
1793  *
1794  * We can't assume that any compression will take place (worst case),
1795  * so the compressed buffer has to be the same size as the uncompressed
1796  * one.  It also must reside (along with the line length buffer) in
1797  * stolen memory.
1798  *
1799  * We need to enable/disable FBC on a global basis.
1800  */
1801 static void intel_update_fbc(struct drm_device *dev)
1802 {
1803         struct drm_i915_private *dev_priv = dev->dev_private;
1804         struct drm_crtc *crtc = NULL, *tmp_crtc;
1805         struct intel_crtc *intel_crtc;
1806         struct drm_framebuffer *fb;
1807         struct intel_framebuffer *intel_fb;
1808         struct drm_i915_gem_object *obj;
1809         int enable_fbc;
1810
1811         DRM_DEBUG_KMS("\n");
1812
1813         if (!i915_powersave)
1814                 return;
1815
1816         if (!I915_HAS_FBC(dev))
1817                 return;
1818
1819         /*
1820          * If FBC is already on, we just have to verify that we can
1821          * keep it that way...
1822          * Need to disable if:
1823          *   - more than one pipe is active
1824          *   - changing FBC params (stride, fence, mode)
1825          *   - new fb is too large to fit in compressed buffer
1826          *   - going to an unsupported config (interlace, pixel multiply, etc.)
1827          */
1828         list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
1829                 if (tmp_crtc->enabled && tmp_crtc->fb) {
1830                         if (crtc) {
1831                                 DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
1832                                 dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
1833                                 goto out_disable;
1834                         }
1835                         crtc = tmp_crtc;
1836                 }
1837         }
1838
1839         if (!crtc || crtc->fb == NULL) {
1840                 DRM_DEBUG_KMS("no output, disabling\n");
1841                 dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
1842                 goto out_disable;
1843         }
1844
1845         intel_crtc = to_intel_crtc(crtc);
1846         fb = crtc->fb;
1847         intel_fb = to_intel_framebuffer(fb);
1848         obj = intel_fb->obj;
1849
1850         enable_fbc = i915_enable_fbc;
1851         if (enable_fbc < 0) {
1852                 DRM_DEBUG_KMS("fbc set to per-chip default\n");
1853                 enable_fbc = 1;
1854                 if (INTEL_INFO(dev)->gen <= 5)
1855                         enable_fbc = 0;
1856         }
1857         if (!enable_fbc) {
1858                 DRM_DEBUG_KMS("fbc disabled per module param\n");
1859                 dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
1860                 goto out_disable;
1861         }
1862         if (intel_fb->obj->base.size > dev_priv->cfb_size) {
1863                 DRM_DEBUG_KMS("framebuffer too large, disabling "
1864                               "compression\n");
1865                 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
1866                 goto out_disable;
1867         }
1868         if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) ||
1869             (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
1870                 DRM_DEBUG_KMS("mode incompatible with compression, "
1871                               "disabling\n");
1872                 dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
1873                 goto out_disable;
1874         }
1875         if ((crtc->mode.hdisplay > 2048) ||
1876             (crtc->mode.vdisplay > 1536)) {
1877                 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
1878                 dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
1879                 goto out_disable;
1880         }
1881         if ((IS_I915GM(dev) || IS_I945GM(dev)) && intel_crtc->plane != 0) {
1882                 DRM_DEBUG_KMS("plane not 0, disabling compression\n");
1883                 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
1884                 goto out_disable;
1885         }
1886
1887         /* The use of a CPU fence is mandatory in order to detect writes
1888          * by the CPU to the scanout and trigger updates to the FBC.
1889          */
1890         if (obj->tiling_mode != I915_TILING_X ||
1891             obj->fence_reg == I915_FENCE_REG_NONE) {
1892                 DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
1893                 dev_priv->no_fbc_reason = FBC_NOT_TILED;
1894                 goto out_disable;
1895         }
1896
1897         /* If the kernel debugger is active, always disable compression */
1898         if (in_dbg_master())
1899                 goto out_disable;
1900
1901         /* If the scanout has not changed, don't modify the FBC settings.
1902          * Note that we make the fundamental assumption that the fb->obj
1903          * cannot be unpinned (and have its GTT offset and fence revoked)
1904          * without first being decoupled from the scanout and FBC disabled.
1905          */
1906         if (dev_priv->cfb_plane == intel_crtc->plane &&
1907             dev_priv->cfb_fb == fb->base.id &&
1908             dev_priv->cfb_y == crtc->y)
1909                 return;
1910
1911         if (intel_fbc_enabled(dev)) {
1912                 /* We update FBC along two paths, after changing fb/crtc
1913                  * configuration (modeswitching) and after page-flipping
1914                  * finishes. For the latter, we know that not only did
1915                  * we disable the FBC at the start of the page-flip
1916                  * sequence, but also more than one vblank has passed.
1917                  *
1918                  * For the former case of modeswitching, it is possible
1919                  * to switch between two FBC valid configurations
1920                  * instantaneously so we do need to disable the FBC
1921                  * before we can modify its control registers. We also
1922                  * have to wait for the next vblank for that to take
1923                  * effect. However, since we delay enabling FBC we can
1924                  * assume that a vblank has passed since disabling and
1925                  * that we can safely alter the registers in the deferred
1926                  * callback.
1927                  *
1928                  * In the scenario that we go from a valid to invalid
1929                  * and then back to valid FBC configuration we have
1930                  * no strict enforcement that a vblank occurred since
1931                  * disabling the FBC. However, along all current pipe
1932                  * disabling paths we do need to wait for a vblank at
1933                  * some point. And we wait before enabling FBC anyway.
1934                  */
1935                 DRM_DEBUG_KMS("disabling active FBC for update\n");
1936                 intel_disable_fbc(dev);
1937         }
1938
1939         intel_enable_fbc(crtc, 500);
1940         return;
1941
1942 out_disable:
1943         /* Multiple disables should be harmless */
1944         if (intel_fbc_enabled(dev)) {
1945                 DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
1946                 intel_disable_fbc(dev);
1947         }
1948 }
1949
1950 int
1951 intel_pin_and_fence_fb_obj(struct drm_device *dev,
1952                            struct drm_i915_gem_object *obj,
1953                            struct intel_ring_buffer *pipelined)
1954 {
1955         struct drm_i915_private *dev_priv = dev->dev_private;
1956         u32 alignment;
1957         int ret;
1958
1959         switch (obj->tiling_mode) {
1960         case I915_TILING_NONE:
1961                 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
1962                         alignment = 128 * 1024;
1963                 else if (INTEL_INFO(dev)->gen >= 4)
1964                         alignment = 4 * 1024;
1965                 else
1966                         alignment = 64 * 1024;
1967                 break;
1968         case I915_TILING_X:
1969                 /* pin() will align the object as required by fence */
1970                 alignment = 0;
1971                 break;
1972         case I915_TILING_Y:
1973                 /* FIXME: Is this true? */
1974                 DRM_ERROR("Y tiled not allowed for scan out buffers\n");
1975                 return -EINVAL;
1976         default:
1977                 BUG();
1978         }
1979
1980         dev_priv->mm.interruptible = false;
1981         ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined);
1982         if (ret)
1983                 goto err_interruptible;
1984
1985         /* Install a fence for tiled scan-out. Pre-i965 always needs a
1986          * fence, whereas 965+ only requires a fence if using
1987          * framebuffer compression.  For simplicity, we always install
1988          * a fence as the cost is not that onerous.
1989          */
1990         if (obj->tiling_mode != I915_TILING_NONE) {
1991                 ret = i915_gem_object_get_fence(obj, pipelined);
1992                 if (ret)
1993                         goto err_unpin;
1994         }
1995
1996         dev_priv->mm.interruptible = true;
1997         return 0;
1998
1999 err_unpin:
2000         i915_gem_object_unpin(obj);
2001 err_interruptible:
2002         dev_priv->mm.interruptible = true;
2003         return ret;
2004 }
2005
2006 static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
2007                              int x, int y)
2008 {
2009         struct drm_device *dev = crtc->dev;
2010         struct drm_i915_private *dev_priv = dev->dev_private;
2011         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2012         struct intel_framebuffer *intel_fb;
2013         struct drm_i915_gem_object *obj;
2014         int plane = intel_crtc->plane;
2015         unsigned long Start, Offset;
2016         u32 dspcntr;
2017         u32 reg;
2018
2019         switch (plane) {
2020         case 0:
2021         case 1:
2022                 break;
2023         default:
2024                 DRM_ERROR("Can't update plane %d in SAREA\n", plane);
2025                 return -EINVAL;
2026         }
2027
2028         intel_fb = to_intel_framebuffer(fb);
2029         obj = intel_fb->obj;
2030
2031         reg = DSPCNTR(plane);
2032         dspcntr = I915_READ(reg);
2033         /* Mask out pixel format bits in case we change it */
2034         dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
2035         switch (fb->bits_per_pixel) {
2036         case 8:
2037                 dspcntr |= DISPPLANE_8BPP;
2038                 break;
2039         case 16:
2040                 if (fb->depth == 15)
2041                         dspcntr |= DISPPLANE_15_16BPP;
2042                 else
2043                         dspcntr |= DISPPLANE_16BPP;
2044                 break;
2045         case 24:
2046         case 32:
2047                 dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
2048                 break;
2049         default:
2050                 DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
2051                 return -EINVAL;
2052         }
2053         if (INTEL_INFO(dev)->gen >= 4) {
2054                 if (obj->tiling_mode != I915_TILING_NONE)
2055                         dspcntr |= DISPPLANE_TILED;
2056                 else
2057                         dspcntr &= ~DISPPLANE_TILED;
2058         }
2059
2060         I915_WRITE(reg, dspcntr);
2061
2062         Start = obj->gtt_offset;
2063         Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8);
2064
2065         DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
2066                       Start, Offset, x, y, fb->pitch);
2067         I915_WRITE(DSPSTRIDE(plane), fb->pitch);
2068         if (INTEL_INFO(dev)->gen >= 4) {
2069                 I915_WRITE(DSPSURF(plane), Start);
2070                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2071                 I915_WRITE(DSPADDR(plane), Offset);
2072         } else
2073                 I915_WRITE(DSPADDR(plane), Start + Offset);
2074         POSTING_READ(reg);
2075
2076         return 0;
2077 }
2078
2079 static int ironlake_update_plane(struct drm_crtc *crtc,
2080                                  struct drm_framebuffer *fb, int x, int y)
2081 {
2082         struct drm_device *dev = crtc->dev;
2083         struct drm_i915_private *dev_priv = dev->dev_private;
2084         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2085         struct intel_framebuffer *intel_fb;
2086         struct drm_i915_gem_object *obj;
2087         int plane = intel_crtc->plane;
2088         unsigned long Start, Offset;
2089         u32 dspcntr;
2090         u32 reg;
2091
2092         switch (plane) {
2093         case 0:
2094         case 1:
2095         case 2:
2096                 break;
2097         default:
2098                 DRM_ERROR("Can't update plane %d in SAREA\n", plane);
2099                 return -EINVAL;
2100         }
2101
2102         intel_fb = to_intel_framebuffer(fb);
2103         obj = intel_fb->obj;
2104
2105         reg = DSPCNTR(plane);
2106         dspcntr = I915_READ(reg);
2107         /* Mask out pixel format bits in case we change it */
2108         dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
2109         switch (fb->bits_per_pixel) {
2110         case 8:
2111                 dspcntr |= DISPPLANE_8BPP;
2112                 break;
2113         case 16:
2114                 if (fb->depth != 16)
2115                         return -EINVAL;
2116
2117                 dspcntr |= DISPPLANE_16BPP;
2118                 break;
2119         case 24:
2120         case 32:
2121                 if (fb->depth == 24)
2122                         dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
2123                 else if (fb->depth == 30)
2124                         dspcntr |= DISPPLANE_32BPP_30BIT_NO_ALPHA;
2125                 else
2126                         return -EINVAL;
2127                 break;
2128         default:
2129                 DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
2130                 return -EINVAL;
2131         }
2132
2133         if (obj->tiling_mode != I915_TILING_NONE)
2134                 dspcntr |= DISPPLANE_TILED;
2135         else
2136                 dspcntr &= ~DISPPLANE_TILED;
2137
2138         /* must disable */
2139         dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2140
2141         I915_WRITE(reg, dspcntr);
2142
2143         Start = obj->gtt_offset;
2144         Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8);
2145
2146         DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
2147                       Start, Offset, x, y, fb->pitch);
2148         I915_WRITE(DSPSTRIDE(plane), fb->pitch);
2149         I915_WRITE(DSPSURF(plane), Start);
2150         I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2151         I915_WRITE(DSPADDR(plane), Offset);
2152         POSTING_READ(reg);
2153
2154         return 0;
2155 }
2156
2157 /* Assume fb object is pinned & idle & fenced and just update base pointers */
2158 static int
2159 intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
2160                            int x, int y, enum mode_set_atomic state)
2161 {
2162         struct drm_device *dev = crtc->dev;
2163         struct drm_i915_private *dev_priv = dev->dev_private;
2164         int ret;
2165
2166         ret = dev_priv->display.update_plane(crtc, fb, x, y);
2167         if (ret)
2168                 return ret;
2169
2170         intel_update_fbc(dev);
2171         intel_increase_pllclock(crtc);
2172
2173         return 0;
2174 }
2175
2176 static int
2177 intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
2178                     struct drm_framebuffer *old_fb)
2179 {
2180         struct drm_device *dev = crtc->dev;
2181         struct drm_i915_master_private *master_priv;
2182         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2183         int ret;
2184
2185         /* no fb bound */
2186         if (!crtc->fb) {
2187                 DRM_ERROR("No FB bound\n");
2188                 return 0;
2189         }
2190
2191         switch (intel_crtc->plane) {
2192         case 0:
2193         case 1:
2194                 break;
2195         case 2:
2196                 if (IS_IVYBRIDGE(dev))
2197                         break;
2198                 /* fall through otherwise */
2199         default:
2200                 DRM_ERROR("no plane for crtc\n");
2201                 return -EINVAL;
2202         }
2203
2204         mutex_lock(&dev->struct_mutex);
2205         ret = intel_pin_and_fence_fb_obj(dev,
2206                                          to_intel_framebuffer(crtc->fb)->obj,
2207                                          NULL);
2208         if (ret != 0) {
2209                 mutex_unlock(&dev->struct_mutex);
2210                 DRM_ERROR("pin & fence failed\n");
2211                 return ret;
2212         }
2213
2214         if (old_fb) {
2215                 struct drm_i915_private *dev_priv = dev->dev_private;
2216                 struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj;
2217
2218                 wait_event(dev_priv->pending_flip_queue,
2219                            atomic_read(&dev_priv->mm.wedged) ||
2220                            atomic_read(&obj->pending_flip) == 0);
2221
2222                 /* Big Hammer, we also need to ensure that any pending
2223                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
2224                  * current scanout is retired before unpinning the old
2225                  * framebuffer.
2226                  *
2227                  * This should only fail upon a hung GPU, in which case we
2228                  * can safely continue.
2229                  */
2230                 ret = i915_gem_object_finish_gpu(obj);
2231                 (void) ret;
2232         }
2233
2234         ret = intel_pipe_set_base_atomic(crtc, crtc->fb, x, y,
2235                                          LEAVE_ATOMIC_MODE_SET);
2236         if (ret) {
2237                 i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
2238                 mutex_unlock(&dev->struct_mutex);
2239                 DRM_ERROR("failed to update base address\n");
2240                 return ret;
2241         }
2242
2243         if (old_fb) {
2244                 intel_wait_for_vblank(dev, intel_crtc->pipe);
2245                 i915_gem_object_unpin(to_intel_framebuffer(old_fb)->obj);
2246         }
2247
2248         mutex_unlock(&dev->struct_mutex);
2249
2250         if (!dev->primary->master)
2251                 return 0;
2252
2253         master_priv = dev->primary->master->driver_priv;
2254         if (!master_priv->sarea_priv)
2255                 return 0;
2256
2257         if (intel_crtc->pipe) {
2258                 master_priv->sarea_priv->pipeB_x = x;
2259                 master_priv->sarea_priv->pipeB_y = y;
2260         } else {
2261                 master_priv->sarea_priv->pipeA_x = x;
2262                 master_priv->sarea_priv->pipeA_y = y;
2263         }
2264
2265         return 0;
2266 }
2267
2268 static void ironlake_set_pll_edp(struct drm_crtc *crtc, int clock)
2269 {
2270         struct drm_device *dev = crtc->dev;
2271         struct drm_i915_private *dev_priv = dev->dev_private;
2272         u32 dpa_ctl;
2273
2274         DRM_DEBUG_KMS("eDP PLL enable for clock %d\n", clock);
2275         dpa_ctl = I915_READ(DP_A);
2276         dpa_ctl &= ~DP_PLL_FREQ_MASK;
2277
2278         if (clock < 200000) {
2279                 u32 temp;
2280                 dpa_ctl |= DP_PLL_FREQ_160MHZ;
2281                 /* workaround for 160Mhz:
2282                    1) program 0x4600c bits 15:0 = 0x8124
2283                    2) program 0x46010 bit 0 = 1
2284                    3) program 0x46034 bit 24 = 1
2285                    4) program 0x64000 bit 14 = 1
2286                    */
2287                 temp = I915_READ(0x4600c);
2288                 temp &= 0xffff0000;
2289                 I915_WRITE(0x4600c, temp | 0x8124);
2290
2291                 temp = I915_READ(0x46010);
2292                 I915_WRITE(0x46010, temp | 1);
2293
2294                 temp = I915_READ(0x46034);
2295                 I915_WRITE(0x46034, temp | (1 << 24));
2296         } else {
2297                 dpa_ctl |= DP_PLL_FREQ_270MHZ;
2298         }
2299         I915_WRITE(DP_A, dpa_ctl);
2300
2301         POSTING_READ(DP_A);
2302         udelay(500);
2303 }
2304
2305 static void intel_fdi_normal_train(struct drm_crtc *crtc)
2306 {
2307         struct drm_device *dev = crtc->dev;
2308         struct drm_i915_private *dev_priv = dev->dev_private;
2309         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2310         int pipe = intel_crtc->pipe;
2311         u32 reg, temp;
2312
2313         /* enable normal train */
2314         reg = FDI_TX_CTL(pipe);
2315         temp = I915_READ(reg);
2316         if (IS_IVYBRIDGE(dev)) {
2317                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
2318                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
2319         } else {
2320                 temp &= ~FDI_LINK_TRAIN_NONE;
2321                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
2322         }
2323         I915_WRITE(reg, temp);
2324
2325         reg = FDI_RX_CTL(pipe);
2326         temp = I915_READ(reg);
2327         if (HAS_PCH_CPT(dev)) {
2328                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2329                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
2330         } else {
2331                 temp &= ~FDI_LINK_TRAIN_NONE;
2332                 temp |= FDI_LINK_TRAIN_NONE;
2333         }
2334         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
2335
2336         /* wait one idle pattern time */
2337         POSTING_READ(reg);
2338         udelay(1000);
2339
2340         /* IVB wants error correction enabled */
2341         if (IS_IVYBRIDGE(dev))
2342                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
2343                            FDI_FE_ERRC_ENABLE);
2344 }
2345
2346 static void cpt_phase_pointer_enable(struct drm_device *dev, int pipe)
2347 {
2348         struct drm_i915_private *dev_priv = dev->dev_private;
2349         u32 flags = I915_READ(SOUTH_CHICKEN1);
2350
2351         flags |= FDI_PHASE_SYNC_OVR(pipe);
2352         I915_WRITE(SOUTH_CHICKEN1, flags); /* once to unlock... */
2353         flags |= FDI_PHASE_SYNC_EN(pipe);
2354         I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to enable */
2355         POSTING_READ(SOUTH_CHICKEN1);
2356 }
2357
2358 /* The FDI link training functions for ILK/Ibexpeak. */
2359 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
2360 {
2361         struct drm_device *dev = crtc->dev;
2362         struct drm_i915_private *dev_priv = dev->dev_private;
2363         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2364         int pipe = intel_crtc->pipe;
2365         int plane = intel_crtc->plane;
2366         u32 reg, temp, tries;
2367
2368         /* FDI needs bits from pipe & plane first */
2369         assert_pipe_enabled(dev_priv, pipe);
2370         assert_plane_enabled(dev_priv, plane);
2371
2372         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
2373            for train result */
2374         reg = FDI_RX_IMR(pipe);
2375         temp = I915_READ(reg);
2376         temp &= ~FDI_RX_SYMBOL_LOCK;
2377         temp &= ~FDI_RX_BIT_LOCK;
2378         I915_WRITE(reg, temp);
2379         I915_READ(reg);
2380         udelay(150);
2381
2382         /* enable CPU FDI TX and PCH FDI RX */
2383         reg = FDI_TX_CTL(pipe);
2384         temp = I915_READ(reg);
2385         temp &= ~(7 << 19);
2386         temp |= (intel_crtc->fdi_lanes - 1) << 19;
2387         temp &= ~FDI_LINK_TRAIN_NONE;
2388         temp |= FDI_LINK_TRAIN_PATTERN_1;
2389         I915_WRITE(reg, temp | FDI_TX_ENABLE);
2390
2391         reg = FDI_RX_CTL(pipe);
2392         temp = I915_READ(reg);
2393         temp &= ~FDI_LINK_TRAIN_NONE;
2394         temp |= FDI_LINK_TRAIN_PATTERN_1;
2395         I915_WRITE(reg, temp | FDI_RX_ENABLE);
2396
2397         POSTING_READ(reg);
2398         udelay(150);
2399
2400         /* Ironlake workaround, enable clock pointer after FDI enable*/
2401         if (HAS_PCH_IBX(dev)) {
2402                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
2403                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
2404                            FDI_RX_PHASE_SYNC_POINTER_EN);
2405         }
2406
2407         reg = FDI_RX_IIR(pipe);
2408         for (tries = 0; tries < 5; tries++) {
2409                 temp = I915_READ(reg);
2410                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2411
2412                 if ((temp & FDI_RX_BIT_LOCK)) {
2413                         DRM_DEBUG_KMS("FDI train 1 done.\n");
2414                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
2415                         break;
2416                 }
2417         }
2418         if (tries == 5)
2419                 DRM_ERROR("FDI train 1 fail!\n");
2420
2421         /* Train 2 */
2422         reg = FDI_TX_CTL(pipe);
2423         temp = I915_READ(reg);
2424         temp &= ~FDI_LINK_TRAIN_NONE;
2425         temp |= FDI_LINK_TRAIN_PATTERN_2;
2426         I915_WRITE(reg, temp);
2427
2428         reg = FDI_RX_CTL(pipe);
2429         temp = I915_READ(reg);
2430         temp &= ~FDI_LINK_TRAIN_NONE;
2431         temp |= FDI_LINK_TRAIN_PATTERN_2;
2432         I915_WRITE(reg, temp);
2433
2434         POSTING_READ(reg);
2435         udelay(150);
2436
2437         reg = FDI_RX_IIR(pipe);
2438         for (tries = 0; tries < 5; tries++) {
2439                 temp = I915_READ(reg);
2440                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2441
2442                 if (temp & FDI_RX_SYMBOL_LOCK) {
2443                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
2444                         DRM_DEBUG_KMS("FDI train 2 done.\n");
2445                         break;
2446                 }
2447         }
2448         if (tries == 5)
2449                 DRM_ERROR("FDI train 2 fail!\n");
2450
2451         DRM_DEBUG_KMS("FDI train done\n");
2452
2453 }
2454
2455 static const int snb_b_fdi_train_param[] = {
2456         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
2457         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
2458         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
2459         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
2460 };
2461
2462 /* The FDI link training functions for SNB/Cougarpoint. */
2463 static void gen6_fdi_link_train(struct drm_crtc *crtc)
2464 {
2465         struct drm_device *dev = crtc->dev;
2466         struct drm_i915_private *dev_priv = dev->dev_private;
2467         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2468         int pipe = intel_crtc->pipe;
2469         u32 reg, temp, i;
2470
2471         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
2472            for train result */
2473         reg = FDI_RX_IMR(pipe);
2474         temp = I915_READ(reg);
2475         temp &= ~FDI_RX_SYMBOL_LOCK;
2476         temp &= ~FDI_RX_BIT_LOCK;
2477         I915_WRITE(reg, temp);
2478
2479         POSTING_READ(reg);
2480         udelay(150);
2481
2482         /* enable CPU FDI TX and PCH FDI RX */
2483         reg = FDI_TX_CTL(pipe);
2484         temp = I915_READ(reg);
2485         temp &= ~(7 << 19);
2486         temp |= (intel_crtc->fdi_lanes - 1) << 19;
2487         temp &= ~FDI_LINK_TRAIN_NONE;
2488         temp |= FDI_LINK_TRAIN_PATTERN_1;
2489         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2490         /* SNB-B */
2491         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
2492         I915_WRITE(reg, temp | FDI_TX_ENABLE);
2493
2494         reg = FDI_RX_CTL(pipe);
2495         temp = I915_READ(reg);
2496         if (HAS_PCH_CPT(dev)) {
2497                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2498                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
2499         } else {
2500                 temp &= ~FDI_LINK_TRAIN_NONE;
2501                 temp |= FDI_LINK_TRAIN_PATTERN_1;
2502         }
2503         I915_WRITE(reg, temp | FDI_RX_ENABLE);
2504
2505         POSTING_READ(reg);
2506         udelay(150);
2507
2508         if (HAS_PCH_CPT(dev))
2509                 cpt_phase_pointer_enable(dev, pipe);
2510
2511         for (i = 0; i < 4; i++) {
2512                 reg = FDI_TX_CTL(pipe);
2513                 temp = I915_READ(reg);
2514                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2515                 temp |= snb_b_fdi_train_param[i];
2516                 I915_WRITE(reg, temp);
2517
2518                 POSTING_READ(reg);
2519                 udelay(500);
2520
2521                 reg = FDI_RX_IIR(pipe);
2522                 temp = I915_READ(reg);
2523                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2524
2525                 if (temp & FDI_RX_BIT_LOCK) {
2526                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
2527                         DRM_DEBUG_KMS("FDI train 1 done.\n");
2528                         break;
2529                 }
2530         }
2531         if (i == 4)
2532                 DRM_ERROR("FDI train 1 fail!\n");
2533
2534         /* Train 2 */
2535         reg = FDI_TX_CTL(pipe);
2536         temp = I915_READ(reg);
2537         temp &= ~FDI_LINK_TRAIN_NONE;
2538         temp |= FDI_LINK_TRAIN_PATTERN_2;
2539         if (IS_GEN6(dev)) {
2540                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2541                 /* SNB-B */
2542                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
2543         }
2544         I915_WRITE(reg, temp);
2545
2546         reg = FDI_RX_CTL(pipe);
2547         temp = I915_READ(reg);
2548         if (HAS_PCH_CPT(dev)) {
2549                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2550                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
2551         } else {
2552                 temp &= ~FDI_LINK_TRAIN_NONE;
2553                 temp |= FDI_LINK_TRAIN_PATTERN_2;
2554         }
2555         I915_WRITE(reg, temp);
2556
2557         POSTING_READ(reg);
2558         udelay(150);
2559
2560         for (i = 0; i < 4; i++) {
2561                 reg = FDI_TX_CTL(pipe);
2562                 temp = I915_READ(reg);
2563                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2564                 temp |= snb_b_fdi_train_param[i];
2565                 I915_WRITE(reg, temp);
2566
2567                 POSTING_READ(reg);
2568                 udelay(500);
2569
2570                 reg = FDI_RX_IIR(pipe);
2571                 temp = I915_READ(reg);
2572                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2573
2574                 if (temp & FDI_RX_SYMBOL_LOCK) {
2575                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
2576                         DRM_DEBUG_KMS("FDI train 2 done.\n");
2577                         break;
2578                 }
2579         }
2580         if (i == 4)
2581                 DRM_ERROR("FDI train 2 fail!\n");
2582
2583         DRM_DEBUG_KMS("FDI train done.\n");
2584 }
2585
2586 /* Manual link training for Ivy Bridge A0 parts */
2587 static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
2588 {
2589         struct drm_device *dev = crtc->dev;
2590         struct drm_i915_private *dev_priv = dev->dev_private;
2591         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2592         int pipe = intel_crtc->pipe;
2593         u32 reg, temp, i;
2594
2595         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
2596            for train result */
2597         reg = FDI_RX_IMR(pipe);
2598         temp = I915_READ(reg);
2599         temp &= ~FDI_RX_SYMBOL_LOCK;
2600         temp &= ~FDI_RX_BIT_LOCK;
2601         I915_WRITE(reg, temp);
2602
2603         POSTING_READ(reg);
2604         udelay(150);
2605
2606         /* enable CPU FDI TX and PCH FDI RX */
2607         reg = FDI_TX_CTL(pipe);
2608         temp = I915_READ(reg);
2609         temp &= ~(7 << 19);
2610         temp |= (intel_crtc->fdi_lanes - 1) << 19;
2611         temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
2612         temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
2613         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2614         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
2615         temp |= FDI_COMPOSITE_SYNC;
2616         I915_WRITE(reg, temp | FDI_TX_ENABLE);
2617
2618         reg = FDI_RX_CTL(pipe);
2619         temp = I915_READ(reg);
2620         temp &= ~FDI_LINK_TRAIN_AUTO;
2621         temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2622         temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
2623         temp |= FDI_COMPOSITE_SYNC;
2624         I915_WRITE(reg, temp | FDI_RX_ENABLE);
2625
2626         POSTING_READ(reg);
2627         udelay(150);
2628
2629         if (HAS_PCH_CPT(dev))
2630                 cpt_phase_pointer_enable(dev, pipe);
2631
2632         for (i = 0; i < 4; i++) {
2633                 reg = FDI_TX_CTL(pipe);
2634                 temp = I915_READ(reg);
2635                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2636                 temp |= snb_b_fdi_train_param[i];
2637                 I915_WRITE(reg, temp);
2638
2639                 POSTING_READ(reg);
2640                 udelay(500);
2641
2642                 reg = FDI_RX_IIR(pipe);
2643                 temp = I915_READ(reg);
2644                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2645
2646                 if (temp & FDI_RX_BIT_LOCK ||
2647                     (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
2648                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
2649                         DRM_DEBUG_KMS("FDI train 1 done.\n");
2650                         break;
2651                 }
2652         }
2653         if (i == 4)
2654                 DRM_ERROR("FDI train 1 fail!\n");
2655
2656         /* Train 2 */
2657         reg = FDI_TX_CTL(pipe);
2658         temp = I915_READ(reg);
2659         temp &= ~FDI_LINK_TRAIN_NONE_IVB;
2660         temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
2661         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2662         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
2663         I915_WRITE(reg, temp);
2664
2665         reg = FDI_RX_CTL(pipe);
2666         temp = I915_READ(reg);
2667         temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2668         temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
2669         I915_WRITE(reg, temp);
2670
2671         POSTING_READ(reg);
2672         udelay(150);
2673
2674         for (i = 0; i < 4; i++) {
2675                 reg = FDI_TX_CTL(pipe);
2676                 temp = I915_READ(reg);
2677                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2678                 temp |= snb_b_fdi_train_param[i];
2679                 I915_WRITE(reg, temp);
2680
2681                 POSTING_READ(reg);
2682                 udelay(500);
2683
2684                 reg = FDI_RX_IIR(pipe);
2685                 temp = I915_READ(reg);
2686                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2687
2688                 if (temp & FDI_RX_SYMBOL_LOCK) {
2689                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
2690                         DRM_DEBUG_KMS("FDI train 2 done.\n");
2691                         break;
2692                 }
2693         }
2694         if (i == 4)
2695                 DRM_ERROR("FDI train 2 fail!\n");
2696
2697         DRM_DEBUG_KMS("FDI train done.\n");
2698 }
2699
2700 static void ironlake_fdi_pll_enable(struct drm_crtc *crtc)
2701 {
2702         struct drm_device *dev = crtc->dev;
2703         struct drm_i915_private *dev_priv = dev->dev_private;
2704         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2705         int pipe = intel_crtc->pipe;
2706         u32 reg, temp;
2707
2708         /* Write the TU size bits so error detection works */
2709         I915_WRITE(FDI_RX_TUSIZE1(pipe),
2710                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
2711
2712         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
2713         reg = FDI_RX_CTL(pipe);
2714         temp = I915_READ(reg);
2715         temp &= ~((0x7 << 19) | (0x7 << 16));
2716         temp |= (intel_crtc->fdi_lanes - 1) << 19;
2717         temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
2718         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
2719
2720         POSTING_READ(reg);
2721         udelay(200);
2722
2723         /* Switch from Rawclk to PCDclk */
2724         temp = I915_READ(reg);
2725         I915_WRITE(reg, temp | FDI_PCDCLK);
2726
2727         POSTING_READ(reg);
2728         udelay(200);
2729
2730         /* Enable CPU FDI TX PLL, always on for Ironlake */
2731         reg = FDI_TX_CTL(pipe);
2732         temp = I915_READ(reg);
2733         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
2734                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
2735
2736                 POSTING_READ(reg);
2737                 udelay(100);
2738         }
2739 }
2740
2741 static void cpt_phase_pointer_disable(struct drm_device *dev, int pipe)
2742 {
2743         struct drm_i915_private *dev_priv = dev->dev_private;
2744         u32 flags = I915_READ(SOUTH_CHICKEN1);
2745
2746         flags &= ~(FDI_PHASE_SYNC_EN(pipe));
2747         I915_WRITE(SOUTH_CHICKEN1, flags); /* once to disable... */
2748         flags &= ~(FDI_PHASE_SYNC_OVR(pipe));
2749         I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to lock */
2750         POSTING_READ(SOUTH_CHICKEN1);
2751 }
2752 static void ironlake_fdi_disable(struct drm_crtc *crtc)
2753 {
2754         struct drm_device *dev = crtc->dev;
2755         struct drm_i915_private *dev_priv = dev->dev_private;
2756         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2757         int pipe = intel_crtc->pipe;
2758         u32 reg, temp;
2759
2760         /* disable CPU FDI tx and PCH FDI rx */
2761         reg = FDI_TX_CTL(pipe);
2762         temp = I915_READ(reg);
2763         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
2764         POSTING_READ(reg);
2765
2766         reg = FDI_RX_CTL(pipe);
2767         temp = I915_READ(reg);
2768         temp &= ~(0x7 << 16);
2769         temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
2770         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
2771
2772         POSTING_READ(reg);
2773         udelay(100);
2774
2775         /* Ironlake workaround, disable clock pointer after downing FDI */
2776         if (HAS_PCH_IBX(dev)) {
2777                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
2778                 I915_WRITE(FDI_RX_CHICKEN(pipe),
2779                            I915_READ(FDI_RX_CHICKEN(pipe) &
2780                                      ~FDI_RX_PHASE_SYNC_POINTER_EN));
2781         } else if (HAS_PCH_CPT(dev)) {
2782                 cpt_phase_pointer_disable(dev, pipe);
2783         }
2784
2785         /* still set train pattern 1 */
2786         reg = FDI_TX_CTL(pipe);
2787         temp = I915_READ(reg);
2788         temp &= ~FDI_LINK_TRAIN_NONE;
2789         temp |= FDI_LINK_TRAIN_PATTERN_1;
2790         I915_WRITE(reg, temp);
2791
2792         reg = FDI_RX_CTL(pipe);
2793         temp = I915_READ(reg);
2794         if (HAS_PCH_CPT(dev)) {
2795                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2796                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
2797         } else {
2798                 temp &= ~FDI_LINK_TRAIN_NONE;
2799                 temp |= FDI_LINK_TRAIN_PATTERN_1;
2800         }
2801         /* BPC in FDI rx is consistent with that in PIPECONF */
2802         temp &= ~(0x07 << 16);
2803         temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
2804         I915_WRITE(reg, temp);
2805
2806         POSTING_READ(reg);
2807         udelay(100);
2808 }
2809
2810 /*
2811  * When we disable a pipe, we need to clear any pending scanline wait events
2812  * to avoid hanging the ring, which we assume we are waiting on.
2813  */
2814 static void intel_clear_scanline_wait(struct drm_device *dev)
2815 {
2816         struct drm_i915_private *dev_priv = dev->dev_private;
2817         struct intel_ring_buffer *ring;
2818         u32 tmp;
2819
2820         if (IS_GEN2(dev))
2821                 /* Can't break the hang on i8xx */
2822                 return;
2823
2824         ring = LP_RING(dev_priv);
2825         tmp = I915_READ_CTL(ring);
2826         if (tmp & RING_WAIT)
2827                 I915_WRITE_CTL(ring, tmp);
2828 }
2829
2830 static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
2831 {
2832         struct drm_i915_gem_object *obj;
2833         struct drm_i915_private *dev_priv;
2834
2835         if (crtc->fb == NULL)
2836                 return;
2837
2838         obj = to_intel_framebuffer(crtc->fb)->obj;
2839         dev_priv = crtc->dev->dev_private;
2840         wait_event(dev_priv->pending_flip_queue,
2841                    atomic_read(&obj->pending_flip) == 0);
2842 }
2843
2844 static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
2845 {
2846         struct drm_device *dev = crtc->dev;
2847         struct drm_mode_config *mode_config = &dev->mode_config;
2848         struct intel_encoder *encoder;
2849
2850         /*
2851          * If there's a non-PCH eDP on this crtc, it must be DP_A, and that
2852          * must be driven by its own crtc; no sharing is possible.
2853          */
2854         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
2855                 if (encoder->base.crtc != crtc)
2856                         continue;
2857
2858                 switch (encoder->type) {
2859                 case INTEL_OUTPUT_EDP:
2860                         if (!intel_encoder_is_pch_edp(&encoder->base))
2861                                 return false;
2862                         continue;
2863                 }
2864         }
2865
2866         return true;
2867 }
2868
2869 /*
2870  * Enable PCH resources required for PCH ports:
2871  *   - PCH PLLs
2872  *   - FDI training & RX/TX
2873  *   - update transcoder timings
2874  *   - DP transcoding bits
2875  *   - transcoder
2876  */
2877 static void ironlake_pch_enable(struct drm_crtc *crtc)
2878 {
2879         struct drm_device *dev = crtc->dev;
2880         struct drm_i915_private *dev_priv = dev->dev_private;
2881         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2882         int pipe = intel_crtc->pipe;
2883         u32 reg, temp;
2884
2885         /* For PCH output, training FDI link */
2886         dev_priv->display.fdi_link_train(crtc);
2887
2888         intel_enable_pch_pll(dev_priv, pipe);
2889
2890         if (HAS_PCH_CPT(dev)) {
2891                 /* Be sure PCH DPLL SEL is set */
2892                 temp = I915_READ(PCH_DPLL_SEL);
2893                 if (pipe == 0 && (temp & TRANSA_DPLL_ENABLE) == 0)
2894                         temp |= (TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL);
2895                 else if (pipe == 1 && (temp & TRANSB_DPLL_ENABLE) == 0)
2896                         temp |= (TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
2897                 else if (pipe == 2 && (temp & TRANSC_DPLL_ENABLE) == 0)
2898                         temp |= (TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL);
2899                 I915_WRITE(PCH_DPLL_SEL, temp);
2900         }
2901
2902         /* set transcoder timing, panel must allow it */
2903         assert_panel_unlocked(dev_priv, pipe);
2904         I915_WRITE(TRANS_HTOTAL(pipe), I915_READ(HTOTAL(pipe)));
2905         I915_WRITE(TRANS_HBLANK(pipe), I915_READ(HBLANK(pipe)));
2906         I915_WRITE(TRANS_HSYNC(pipe),  I915_READ(HSYNC(pipe)));
2907
2908         I915_WRITE(TRANS_VTOTAL(pipe), I915_READ(VTOTAL(pipe)));
2909         I915_WRITE(TRANS_VBLANK(pipe), I915_READ(VBLANK(pipe)));
2910         I915_WRITE(TRANS_VSYNC(pipe),  I915_READ(VSYNC(pipe)));
2911
2912         intel_fdi_normal_train(crtc);
2913
2914         /* For PCH DP, enable TRANS_DP_CTL */
2915         if (HAS_PCH_CPT(dev) &&
2916             intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
2917                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) >> 5;
2918                 reg = TRANS_DP_CTL(pipe);
2919                 temp = I915_READ(reg);
2920                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
2921                           TRANS_DP_SYNC_MASK |
2922                           TRANS_DP_BPC_MASK);
2923                 temp |= (TRANS_DP_OUTPUT_ENABLE |
2924                          TRANS_DP_ENH_FRAMING);
2925                 temp |= bpc << 9; /* same format but at 11:9 */
2926
2927                 if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
2928                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
2929                 if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
2930                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
2931
2932                 switch (intel_trans_dp_port_sel(crtc)) {
2933                 case PCH_DP_B:
2934                         temp |= TRANS_DP_PORT_SEL_B;
2935                         break;
2936                 case PCH_DP_C:
2937                         temp |= TRANS_DP_PORT_SEL_C;
2938                         break;
2939                 case PCH_DP_D:
2940                         temp |= TRANS_DP_PORT_SEL_D;
2941                         break;
2942                 default:
2943                         DRM_DEBUG_KMS("Wrong PCH DP port return. Guess port B\n");
2944                         temp |= TRANS_DP_PORT_SEL_B;
2945                         break;
2946                 }
2947
2948                 I915_WRITE(reg, temp);
2949         }
2950
2951         intel_enable_transcoder(dev_priv, pipe);
2952 }
2953
2954 static void ironlake_crtc_enable(struct drm_crtc *crtc)
2955 {
2956         struct drm_device *dev = crtc->dev;
2957         struct drm_i915_private *dev_priv = dev->dev_private;
2958         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2959         int pipe = intel_crtc->pipe;
2960         int plane = intel_crtc->plane;
2961         u32 temp;
2962         bool is_pch_port;
2963
2964         if (intel_crtc->active)
2965                 return;
2966
2967         intel_crtc->active = true;
2968         intel_update_watermarks(dev);
2969
2970         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
2971                 temp = I915_READ(PCH_LVDS);
2972                 if ((temp & LVDS_PORT_EN) == 0)
2973                         I915_WRITE(PCH_LVDS, temp | LVDS_PORT_EN);
2974         }
2975
2976         is_pch_port = intel_crtc_driving_pch(crtc);
2977
2978         if (is_pch_port)
2979                 ironlake_fdi_pll_enable(crtc);
2980         else
2981                 ironlake_fdi_disable(crtc);
2982
2983         /* Enable panel fitting for LVDS */
2984         if (dev_priv->pch_pf_size &&
2985             (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) || HAS_eDP)) {
2986                 /* Force use of hard-coded filter coefficients
2987                  * as some pre-programmed values are broken,
2988                  * e.g. x201.
2989                  */
2990                 I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
2991                 I915_WRITE(PF_WIN_POS(pipe), dev_priv->pch_pf_pos);
2992                 I915_WRITE(PF_WIN_SZ(pipe), dev_priv->pch_pf_size);
2993         }
2994
2995         /*
2996          * On ILK+ LUT must be loaded before the pipe is running but with
2997          * clocks enabled
2998          */
2999         intel_crtc_load_lut(crtc);
3000
3001         intel_enable_pipe(dev_priv, pipe, is_pch_port);
3002         intel_enable_plane(dev_priv, plane, pipe);
3003
3004         if (is_pch_port)
3005                 ironlake_pch_enable(crtc);
3006
3007         mutex_lock(&dev->struct_mutex);
3008         intel_update_fbc(dev);
3009         mutex_unlock(&dev->struct_mutex);
3010
3011         intel_crtc_update_cursor(crtc, true);
3012 }
3013
3014 static void ironlake_crtc_disable(struct drm_crtc *crtc)
3015 {
3016         struct drm_device *dev = crtc->dev;
3017         struct drm_i915_private *dev_priv = dev->dev_private;
3018         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3019         int pipe = intel_crtc->pipe;
3020         int plane = intel_crtc->plane;
3021         u32 reg, temp;
3022
3023         if (!intel_crtc->active)
3024                 return;
3025
3026         intel_crtc_wait_for_pending_flips(crtc);
3027         drm_vblank_off(dev, pipe);
3028         intel_crtc_update_cursor(crtc, false);
3029
3030         intel_disable_plane(dev_priv, plane, pipe);
3031
3032         if (dev_priv->cfb_plane == plane)
3033                 intel_disable_fbc(dev);
3034
3035         intel_disable_pipe(dev_priv, pipe);
3036
3037         /* Disable PF */
3038         I915_WRITE(PF_CTL(pipe), 0);
3039         I915_WRITE(PF_WIN_SZ(pipe), 0);
3040
3041         ironlake_fdi_disable(crtc);
3042
3043         /* This is a horrible layering violation; we should be doing this in
3044          * the connector/encoder ->prepare instead, but we don't always have
3045          * enough information there about the config to know whether it will
3046          * actually be necessary or just cause undesired flicker.
3047          */
3048         intel_disable_pch_ports(dev_priv, pipe);
3049
3050         intel_disable_transcoder(dev_priv, pipe);
3051
3052         if (HAS_PCH_CPT(dev)) {
3053                 /* disable TRANS_DP_CTL */
3054                 reg = TRANS_DP_CTL(pipe);
3055                 temp = I915_READ(reg);
3056                 temp &= ~(TRANS_DP_OUTPUT_ENABLE | TRANS_DP_PORT_SEL_MASK);
3057                 temp |= TRANS_DP_PORT_SEL_NONE;
3058                 I915_WRITE(reg, temp);
3059
3060                 /* disable DPLL_SEL */
3061                 temp = I915_READ(PCH_DPLL_SEL);
3062                 switch (pipe) {
3063                 case 0:
3064                         temp &= ~(TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL);
3065                         break;
3066                 case 1:
3067                         temp &= ~(TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
3068                         break;
3069                 case 2:
3070                         /* FIXME: manage transcoder PLLs? */
3071                         temp &= ~(TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL);
3072                         break;
3073                 default:
3074                         BUG(); /* wtf */
3075                 }
3076                 I915_WRITE(PCH_DPLL_SEL, temp);
3077         }
3078
3079         /* disable PCH DPLL */
3080         intel_disable_pch_pll(dev_priv, pipe);
3081
3082         /* Switch from PCDclk to Rawclk */
3083         reg = FDI_RX_CTL(pipe);
3084         temp = I915_READ(reg);
3085         I915_WRITE(reg, temp & ~FDI_PCDCLK);
3086
3087         /* Disable CPU FDI TX PLL */
3088         reg = FDI_TX_CTL(pipe);
3089         temp = I915_READ(reg);
3090         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
3091
3092         POSTING_READ(reg);
3093         udelay(100);
3094
3095         reg = FDI_RX_CTL(pipe);
3096         temp = I915_READ(reg);
3097         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
3098
3099         /* Wait for the clocks to turn off. */
3100         POSTING_READ(reg);
3101         udelay(100);
3102
3103         intel_crtc->active = false;
3104         intel_update_watermarks(dev);
3105
3106         mutex_lock(&dev->struct_mutex);
3107         intel_update_fbc(dev);
3108         intel_clear_scanline_wait(dev);
3109         mutex_unlock(&dev->struct_mutex);
3110 }
3111
3112 static void ironlake_crtc_dpms(struct drm_crtc *crtc, int mode)
3113 {
3114         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3115         int pipe = intel_crtc->pipe;
3116         int plane = intel_crtc->plane;
3117
3118         /* XXX: When our outputs are all unaware of DPMS modes other than off
3119          * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
3120          */
3121         switch (mode) {
3122         case DRM_MODE_DPMS_ON:
3123         case DRM_MODE_DPMS_STANDBY:
3124         case DRM_MODE_DPMS_SUSPEND:
3125                 DRM_DEBUG_KMS("crtc %d/%d dpms on\n", pipe, plane);
3126                 ironlake_crtc_enable(crtc);
3127                 break;
3128
3129         case DRM_MODE_DPMS_OFF:
3130                 DRM_DEBUG_KMS("crtc %d/%d dpms off\n", pipe, plane);
3131                 ironlake_crtc_disable(crtc);
3132                 break;
3133         }
3134 }
3135
3136 static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable)
3137 {
3138         if (!enable && intel_crtc->overlay) {
3139                 struct drm_device *dev = intel_crtc->base.dev;
3140                 struct drm_i915_private *dev_priv = dev->dev_private;
3141
3142                 mutex_lock(&dev->struct_mutex);
3143                 dev_priv->mm.interruptible = false;
3144                 (void) intel_overlay_switch_off(intel_crtc->overlay);
3145                 dev_priv->mm.interruptible = true;
3146                 mutex_unlock(&dev->struct_mutex);
3147         }
3148
3149         /* Let userspace switch the overlay on again. In most cases userspace
3150          * has to recompute where to put it anyway.
3151          */
3152 }
3153
3154 static void i9xx_crtc_enable(struct drm_crtc *crtc)
3155 {
3156         struct drm_device *dev = crtc->dev;
3157         struct drm_i915_private *dev_priv = dev->dev_private;
3158         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3159         int pipe = intel_crtc->pipe;
3160         int plane = intel_crtc->plane;
3161
3162         if (intel_crtc->active)
3163                 return;
3164
3165         intel_crtc->active = true;
3166         intel_update_watermarks(dev);
3167
3168         intel_enable_pll(dev_priv, pipe);
3169         intel_enable_pipe(dev_priv, pipe, false);
3170         intel_enable_plane(dev_priv, plane, pipe);
3171
3172         intel_crtc_load_lut(crtc);
3173         intel_update_fbc(dev);
3174
3175         /* Give the overlay scaler a chance to enable if it's on this pipe */
3176         intel_crtc_dpms_overlay(intel_crtc, true);
3177         intel_crtc_update_cursor(crtc, true);
3178 }
3179
3180 static void i9xx_crtc_disable(struct drm_crtc *crtc)
3181 {
3182         struct drm_device *dev = crtc->dev;
3183         struct drm_i915_private *dev_priv = dev->dev_private;
3184         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3185         int pipe = intel_crtc->pipe;
3186         int plane = intel_crtc->plane;
3187
3188         if (!intel_crtc->active)
3189                 return;
3190
3191         /* Give the overlay scaler a chance to disable if it's on this pipe */
3192         intel_crtc_wait_for_pending_flips(crtc);
3193         drm_vblank_off(dev, pipe);
3194         intel_crtc_dpms_overlay(intel_crtc, false);
3195         intel_crtc_update_cursor(crtc, false);
3196
3197         if (dev_priv->cfb_plane == plane)
3198                 intel_disable_fbc(dev);
3199
3200         intel_disable_plane(dev_priv, plane, pipe);
3201         intel_disable_pipe(dev_priv, pipe);
3202         intel_disable_pll(dev_priv, pipe);
3203
3204         intel_crtc->active = false;
3205         intel_update_fbc(dev);
3206         intel_update_watermarks(dev);
3207         intel_clear_scanline_wait(dev);
3208 }
3209
3210 static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
3211 {
3212         /* XXX: When our outputs are all unaware of DPMS modes other than off
3213          * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
3214          */
3215         switch (mode) {
3216         case DRM_MODE_DPMS_ON:
3217         case DRM_MODE_DPMS_STANDBY:
3218         case DRM_MODE_DPMS_SUSPEND:
3219                 i9xx_crtc_enable(crtc);
3220                 break;
3221         case DRM_MODE_DPMS_OFF:
3222                 i9xx_crtc_disable(crtc);
3223                 break;
3224         }
3225 }
3226
3227 /**
3228  * Sets the power management mode of the pipe and plane.
3229  */
3230 static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
3231 {
3232         struct drm_device *dev = crtc->dev;
3233         struct drm_i915_private *dev_priv = dev->dev_private;
3234         struct drm_i915_master_private *master_priv;
3235         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3236         int pipe = intel_crtc->pipe;
3237         bool enabled;
3238
3239         if (intel_crtc->dpms_mode == mode)
3240                 return;
3241
3242         intel_crtc->dpms_mode = mode;
3243
3244         dev_priv->display.dpms(crtc, mode);
3245
3246         if (!dev->primary->master)
3247                 return;
3248
3249         master_priv = dev->primary->master->driver_priv;
3250         if (!master_priv->sarea_priv)
3251                 return;
3252
3253         enabled = crtc->enabled && mode != DRM_MODE_DPMS_OFF;
3254
3255         switch (pipe) {
3256         case 0:
3257                 master_priv->sarea_priv->pipeA_w = enabled ? crtc->mode.hdisplay : 0;
3258                 master_priv->sarea_priv->pipeA_h = enabled ? crtc->mode.vdisplay : 0;
3259                 break;
3260         case 1:
3261                 master_priv->sarea_priv->pipeB_w = enabled ? crtc->mode.hdisplay : 0;
3262                 master_priv->sarea_priv->pipeB_h = enabled ? crtc->mode.vdisplay : 0;
3263                 break;
3264         default:
3265                 DRM_ERROR("Can't update pipe %c in SAREA\n", pipe_name(pipe));
3266                 break;
3267         }
3268 }
3269
3270 static void intel_crtc_disable(struct drm_crtc *crtc)
3271 {
3272         struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
3273         struct drm_device *dev = crtc->dev;
3274
3275         crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
3276
3277         if (crtc->fb) {
3278                 mutex_lock(&dev->struct_mutex);
3279                 i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
3280                 mutex_unlock(&dev->struct_mutex);
3281         }
3282 }
3283
3284 /* Prepare for a mode set.
3285  *
3286  * Note we could be a lot smarter here.  We need to figure out which outputs
3287  * will be enabled, which disabled (in short, how the config will changes)
3288  * and perform the minimum necessary steps to accomplish that, e.g. updating
3289  * watermarks, FBC configuration, making sure PLLs are programmed correctly,
3290  * panel fitting is in the proper state, etc.
3291  */
3292 static void i9xx_crtc_prepare(struct drm_crtc *crtc)
3293 {
3294         i9xx_crtc_disable(crtc);
3295 }
3296
3297 static void i9xx_crtc_commit(struct drm_crtc *crtc)
3298 {
3299         i9xx_crtc_enable(crtc);
3300 }
3301
3302 static void ironlake_crtc_prepare(struct drm_crtc *crtc)
3303 {
3304         ironlake_crtc_disable(crtc);
3305 }
3306
3307 static void ironlake_crtc_commit(struct drm_crtc *crtc)
3308 {
3309         ironlake_crtc_enable(crtc);
3310 }
3311
3312 void intel_encoder_prepare(struct drm_encoder *encoder)
3313 {
3314         struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
3315         /* lvds has its own version of prepare see intel_lvds_prepare */
3316         encoder_funcs->dpms(encoder, DRM_MODE_DPMS_OFF);
3317 }
3318
3319 void intel_encoder_commit(struct drm_encoder *encoder)
3320 {
3321         struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
3322         /* lvds has its own version of commit see intel_lvds_commit */
3323         encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
3324 }
3325
3326 void intel_encoder_destroy(struct drm_encoder *encoder)
3327 {
3328         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
3329
3330         drm_encoder_cleanup(encoder);
3331         kfree(intel_encoder);
3332 }
3333
3334 static bool intel_crtc_mode_fixup(struct drm_crtc *crtc,
3335                                   struct drm_display_mode *mode,
3336                                   struct drm_display_mode *adjusted_mode)
3337 {
3338         struct drm_device *dev = crtc->dev;
3339
3340         if (HAS_PCH_SPLIT(dev)) {
3341                 /* FDI link clock is fixed at 2.7G */
3342                 if (mode->clock * 3 > IRONLAKE_FDI_FREQ * 4)
3343                         return false;
3344         }
3345
3346         /* XXX some encoders set the crtcinfo, others don't.
3347          * Obviously we need some form of conflict resolution here...
3348          */
3349         if (adjusted_mode->crtc_htotal == 0)
3350                 drm_mode_set_crtcinfo(adjusted_mode, 0);
3351
3352         return true;
3353 }
3354
3355 static int i945_get_display_clock_speed(struct drm_device *dev)
3356 {
3357         return 400000;
3358 }
3359
3360 static int i915_get_display_clock_speed(struct drm_device *dev)
3361 {
3362         return 333000;
3363 }
3364
3365 static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
3366 {
3367         return 200000;
3368 }
3369
3370 static int i915gm_get_display_clock_speed(struct drm_device *dev)
3371 {
3372         u16 gcfgc = 0;
3373
3374         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
3375
3376         if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
3377                 return 133000;
3378         else {
3379                 switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
3380                 case GC_DISPLAY_CLOCK_333_MHZ:
3381                         return 333000;
3382                 default:
3383                 case GC_DISPLAY_CLOCK_190_200_MHZ:
3384                         return 190000;
3385                 }
3386         }
3387 }
3388
3389 static int i865_get_display_clock_speed(struct drm_device *dev)
3390 {
3391         return 266000;
3392 }
3393
3394 static int i855_get_display_clock_speed(struct drm_device *dev)
3395 {
3396         u16 hpllcc = 0;
3397         /* Assume that the hardware is in the high speed state.  This
3398          * should be the default.
3399          */
3400         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
3401         case GC_CLOCK_133_200:
3402         case GC_CLOCK_100_200:
3403                 return 200000;
3404         case GC_CLOCK_166_250:
3405                 return 250000;
3406         case GC_CLOCK_100_133:
3407                 return 133000;
3408         }
3409
3410         /* Shouldn't happen */
3411         return 0;
3412 }
3413
3414 static int i830_get_display_clock_speed(struct drm_device *dev)
3415 {
3416         return 133000;
3417 }
3418
3419 struct fdi_m_n {
3420         u32        tu;
3421         u32        gmch_m;
3422         u32        gmch_n;
3423         u32        link_m;
3424         u32        link_n;
3425 };
3426
3427 static void
3428 fdi_reduce_ratio(u32 *num, u32 *den)
3429 {
3430         while (*num > 0xffffff || *den > 0xffffff) {
3431                 *num >>= 1;
3432                 *den >>= 1;
3433         }
3434 }
3435
3436 static void
3437 ironlake_compute_m_n(int bits_per_pixel, int nlanes, int pixel_clock,
3438                      int link_clock, struct fdi_m_n *m_n)
3439 {
3440         m_n->tu = 64; /* default size */
3441
3442         /* BUG_ON(pixel_clock > INT_MAX / 36); */
3443         m_n->gmch_m = bits_per_pixel * pixel_clock;
3444         m_n->gmch_n = link_clock * nlanes * 8;
3445         fdi_reduce_ratio(&m_n->gmch_m, &m_n->gmch_n);
3446
3447         m_n->link_m = pixel_clock;
3448         m_n->link_n = link_clock;
3449         fdi_reduce_ratio(&m_n->link_m, &m_n->link_n);
3450 }
3451
3452
3453 struct intel_watermark_params {
3454         unsigned long fifo_size;
3455         unsigned long max_wm;
3456         unsigned long default_wm;
3457         unsigned long guard_size;
3458         unsigned long cacheline_size;
3459 };
3460
3461 /* Pineview has different values for various configs */
3462 static const struct intel_watermark_params pineview_display_wm = {
3463         PINEVIEW_DISPLAY_FIFO,
3464         PINEVIEW_MAX_WM,
3465         PINEVIEW_DFT_WM,
3466         PINEVIEW_GUARD_WM,
3467         PINEVIEW_FIFO_LINE_SIZE
3468 };
3469 static const struct intel_watermark_params pineview_display_hplloff_wm = {
3470         PINEVIEW_DISPLAY_FIFO,
3471         PINEVIEW_MAX_WM,
3472         PINEVIEW_DFT_HPLLOFF_WM,
3473         PINEVIEW_GUARD_WM,
3474         PINEVIEW_FIFO_LINE_SIZE
3475 };
3476 static const struct intel_watermark_params pineview_cursor_wm = {
3477         PINEVIEW_CURSOR_FIFO,
3478         PINEVIEW_CURSOR_MAX_WM,
3479         PINEVIEW_CURSOR_DFT_WM,
3480         PINEVIEW_CURSOR_GUARD_WM,
3481         PINEVIEW_FIFO_LINE_SIZE,
3482 };
3483 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
3484         PINEVIEW_CURSOR_FIFO,
3485         PINEVIEW_CURSOR_MAX_WM,
3486         PINEVIEW_CURSOR_DFT_WM,
3487         PINEVIEW_CURSOR_GUARD_WM,
3488         PINEVIEW_FIFO_LINE_SIZE
3489 };
3490 static const struct intel_watermark_params g4x_wm_info = {
3491         G4X_FIFO_SIZE,
3492         G4X_MAX_WM,
3493         G4X_MAX_WM,
3494         2,
3495         G4X_FIFO_LINE_SIZE,
3496 };
3497 static const struct intel_watermark_params g4x_cursor_wm_info = {
3498         I965_CURSOR_FIFO,
3499         I965_CURSOR_MAX_WM,
3500         I965_CURSOR_DFT_WM,
3501         2,
3502         G4X_FIFO_LINE_SIZE,
3503 };
3504 static const struct intel_watermark_params i965_cursor_wm_info = {
3505         I965_CURSOR_FIFO,
3506         I965_CURSOR_MAX_WM,
3507         I965_CURSOR_DFT_WM,
3508         2,
3509         I915_FIFO_LINE_SIZE,
3510 };
3511 static const struct intel_watermark_params i945_wm_info = {
3512         I945_FIFO_SIZE,
3513         I915_MAX_WM,
3514         1,
3515         2,
3516         I915_FIFO_LINE_SIZE
3517 };
3518 static const struct intel_watermark_params i915_wm_info = {
3519         I915_FIFO_SIZE,
3520         I915_MAX_WM,
3521         1,
3522         2,
3523         I915_FIFO_LINE_SIZE
3524 };
3525 static const struct intel_watermark_params i855_wm_info = {
3526         I855GM_FIFO_SIZE,
3527         I915_MAX_WM,
3528         1,
3529         2,
3530         I830_FIFO_LINE_SIZE
3531 };
3532 static const struct intel_watermark_params i830_wm_info = {
3533         I830_FIFO_SIZE,
3534         I915_MAX_WM,
3535         1,
3536         2,
3537         I830_FIFO_LINE_SIZE
3538 };
3539
3540 static const struct intel_watermark_params ironlake_display_wm_info = {
3541         ILK_DISPLAY_FIFO,
3542         ILK_DISPLAY_MAXWM,
3543         ILK_DISPLAY_DFTWM,
3544         2,
3545         ILK_FIFO_LINE_SIZE
3546 };
3547 static const struct intel_watermark_params ironlake_cursor_wm_info = {
3548         ILK_CURSOR_FIFO,
3549         ILK_CURSOR_MAXWM,
3550         ILK_CURSOR_DFTWM,
3551         2,
3552         ILK_FIFO_LINE_SIZE
3553 };
3554 static const struct intel_watermark_params ironlake_display_srwm_info = {
3555         ILK_DISPLAY_SR_FIFO,
3556         ILK_DISPLAY_MAX_SRWM,
3557         ILK_DISPLAY_DFT_SRWM,
3558         2,
3559         ILK_FIFO_LINE_SIZE
3560 };
3561 static const struct intel_watermark_params ironlake_cursor_srwm_info = {
3562         ILK_CURSOR_SR_FIFO,
3563         ILK_CURSOR_MAX_SRWM,
3564         ILK_CURSOR_DFT_SRWM,
3565         2,
3566         ILK_FIFO_LINE_SIZE
3567 };
3568
3569 static const struct intel_watermark_params sandybridge_display_wm_info = {
3570         SNB_DISPLAY_FIFO,
3571         SNB_DISPLAY_MAXWM,
3572         SNB_DISPLAY_DFTWM,
3573         2,
3574         SNB_FIFO_LINE_SIZE
3575 };
3576 static const struct intel_watermark_params sandybridge_cursor_wm_info = {
3577         SNB_CURSOR_FIFO,
3578         SNB_CURSOR_MAXWM,
3579         SNB_CURSOR_DFTWM,
3580         2,
3581         SNB_FIFO_LINE_SIZE
3582 };
3583 static const struct intel_watermark_params sandybridge_display_srwm_info = {
3584         SNB_DISPLAY_SR_FIFO,
3585         SNB_DISPLAY_MAX_SRWM,
3586         SNB_DISPLAY_DFT_SRWM,
3587         2,
3588         SNB_FIFO_LINE_SIZE
3589 };
3590 static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
3591         SNB_CURSOR_SR_FIFO,
3592         SNB_CURSOR_MAX_SRWM,
3593         SNB_CURSOR_DFT_SRWM,
3594         2,
3595         SNB_FIFO_LINE_SIZE
3596 };
3597
3598
3599 /**
3600  * intel_calculate_wm - calculate watermark level
3601  * @clock_in_khz: pixel clock
3602  * @wm: chip FIFO params
3603  * @pixel_size: display pixel size
3604  * @latency_ns: memory latency for the platform
3605  *
3606  * Calculate the watermark level (the level at which the display plane will
3607  * start fetching from memory again).  Each chip has a different display
3608  * FIFO size and allocation, so the caller needs to figure that out and pass
3609  * in the correct intel_watermark_params structure.
3610  *
3611  * As the pixel clock runs, the FIFO will be drained at a rate that depends
3612  * on the pixel size.  When it reaches the watermark level, it'll start
3613  * fetching FIFO line sized based chunks from memory until the FIFO fills
3614  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
3615  * will occur, and a display engine hang could result.
3616  */
3617 static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
3618                                         const struct intel_watermark_params *wm,
3619                                         int fifo_size,
3620                                         int pixel_size,
3621                                         unsigned long latency_ns)
3622 {
3623         long entries_required, wm_size;
3624
3625         /*
3626          * Note: we need to make sure we don't overflow for various clock &
3627          * latency values.
3628          * clocks go from a few thousand to several hundred thousand.
3629          * latency is usually a few thousand
3630          */
3631         entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
3632                 1000;
3633         entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
3634
3635         DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
3636
3637         wm_size = fifo_size - (entries_required + wm->guard_size);
3638
3639         DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
3640
3641         /* Don't promote wm_size to unsigned... */
3642         if (wm_size > (long)wm->max_wm)
3643                 wm_size = wm->max_wm;
3644         if (wm_size <= 0)
3645                 wm_size = wm->default_wm;
3646         return wm_size;
3647 }
3648
3649 struct cxsr_latency {
3650         int is_desktop;
3651         int is_ddr3;
3652         unsigned long fsb_freq;
3653         unsigned long mem_freq;
3654         unsigned long display_sr;
3655         unsigned long display_hpll_disable;
3656         unsigned long cursor_sr;
3657         unsigned long cursor_hpll_disable;
3658 };
3659
3660 static const struct cxsr_latency cxsr_latency_table[] = {
3661         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
3662         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
3663         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
3664         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
3665         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
3666
3667         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
3668         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
3669         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
3670         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
3671         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
3672
3673         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
3674         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
3675         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
3676         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
3677         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
3678
3679         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
3680         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
3681         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
3682         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
3683         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
3684
3685         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
3686         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
3687         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
3688         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
3689         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
3690
3691         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
3692         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
3693         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
3694         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
3695         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
3696 };
3697
3698 static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
3699                                                          int is_ddr3,
3700                                                          int fsb,
3701                                                          int mem)
3702 {
3703         const struct cxsr_latency *latency;
3704         int i;
3705
3706         if (fsb == 0 || mem == 0)
3707                 return NULL;
3708
3709         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
3710                 latency = &cxsr_latency_table[i];
3711                 if (is_desktop == latency->is_desktop &&
3712                     is_ddr3 == latency->is_ddr3 &&
3713                     fsb == latency->fsb_freq && mem == latency->mem_freq)
3714                         return latency;
3715         }
3716
3717         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
3718
3719         return NULL;
3720 }
3721
3722 static void pineview_disable_cxsr(struct drm_device *dev)
3723 {
3724         struct drm_i915_private *dev_priv = dev->dev_private;
3725
3726         /* deactivate cxsr */
3727         I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
3728 }
3729
3730 /*
3731  * Latency for FIFO fetches is dependent on several factors:
3732  *   - memory configuration (speed, channels)
3733  *   - chipset
3734  *   - current MCH state
3735  * It can be fairly high in some situations, so here we assume a fairly
3736  * pessimal value.  It's a tradeoff between extra memory fetches (if we
3737  * set this value too high, the FIFO will fetch frequently to stay full)
3738  * and power consumption (set it too low to save power and we might see
3739  * FIFO underruns and display "flicker").
3740  *
3741  * A value of 5us seems to be a good balance; safe for very low end
3742  * platforms but not overly aggressive on lower latency configs.
3743  */
3744 static const int latency_ns = 5000;
3745
3746 static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
3747 {
3748         struct drm_i915_private *dev_priv = dev->dev_private;
3749         uint32_t dsparb = I915_READ(DSPARB);
3750         int size;
3751
3752         size = dsparb & 0x7f;
3753         if (plane)
3754                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
3755
3756         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
3757                       plane ? "B" : "A", size);
3758
3759         return size;
3760 }
3761
3762 static int i85x_get_fifo_size(struct drm_device *dev, int plane)
3763 {
3764         struct drm_i915_private *dev_priv = dev->dev_private;
3765         uint32_t dsparb = I915_READ(DSPARB);
3766         int size;
3767
3768         size = dsparb & 0x1ff;
3769         if (plane)
3770                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
3771         size >>= 1; /* Convert to cachelines */
3772
3773         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
3774                       plane ? "B" : "A", size);
3775
3776         return size;
3777 }
3778
3779 static int i845_get_fifo_size(struct drm_device *dev, int plane)
3780 {
3781         struct drm_i915_private *dev_priv = dev->dev_private;
3782         uint32_t dsparb = I915_READ(DSPARB);
3783         int size;
3784
3785         size = dsparb & 0x7f;
3786         size >>= 2; /* Convert to cachelines */
3787
3788         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
3789                       plane ? "B" : "A",
3790                       size);
3791
3792         return size;
3793 }
3794
3795 static int i830_get_fifo_size(struct drm_device *dev, int plane)
3796 {
3797         struct drm_i915_private *dev_priv = dev->dev_private;
3798         uint32_t dsparb = I915_READ(DSPARB);
3799         int size;
3800
3801         size = dsparb & 0x7f;
3802         size >>= 1; /* Convert to cachelines */
3803
3804         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
3805                       plane ? "B" : "A", size);
3806
3807         return size;
3808 }
3809
3810 static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
3811 {
3812         struct drm_crtc *crtc, *enabled = NULL;
3813
3814         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3815                 if (crtc->enabled && crtc->fb) {
3816                         if (enabled)
3817                                 return NULL;
3818                         enabled = crtc;
3819                 }
3820         }
3821
3822         return enabled;
3823 }
3824
3825 static void pineview_update_wm(struct drm_device *dev)
3826 {
3827         struct drm_i915_private *dev_priv = dev->dev_private;
3828         struct drm_crtc *crtc;
3829         const struct cxsr_latency *latency;
3830         u32 reg;
3831         unsigned long wm;
3832
3833         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
3834                                          dev_priv->fsb_freq, dev_priv->mem_freq);
3835         if (!latency) {
3836                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
3837                 pineview_disable_cxsr(dev);
3838                 return;
3839         }
3840
3841         crtc = single_enabled_crtc(dev);
3842         if (crtc) {
3843                 int clock = crtc->mode.clock;
3844                 int pixel_size = crtc->fb->bits_per_pixel / 8;
3845
3846                 /* Display SR */
3847                 wm = intel_calculate_wm(clock, &pineview_display_wm,
3848                                         pineview_display_wm.fifo_size,
3849                                         pixel_size, latency->display_sr);
3850                 reg = I915_READ(DSPFW1);
3851                 reg &= ~DSPFW_SR_MASK;
3852                 reg |= wm << DSPFW_SR_SHIFT;
3853                 I915_WRITE(DSPFW1, reg);
3854                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
3855
3856                 /* cursor SR */
3857                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
3858                                         pineview_display_wm.fifo_size,
3859                                         pixel_size, latency->cursor_sr);
3860                 reg = I915_READ(DSPFW3);
3861                 reg &= ~DSPFW_CURSOR_SR_MASK;
3862                 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
3863                 I915_WRITE(DSPFW3, reg);
3864
3865                 /* Display HPLL off SR */
3866                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
3867                                         pineview_display_hplloff_wm.fifo_size,
3868                                         pixel_size, latency->display_hpll_disable);
3869                 reg = I915_READ(DSPFW3);
3870                 reg &= ~DSPFW_HPLL_SR_MASK;
3871                 reg |= wm & DSPFW_HPLL_SR_MASK;
3872                 I915_WRITE(DSPFW3, reg);
3873
3874                 /* cursor HPLL off SR */
3875                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
3876                                         pineview_display_hplloff_wm.fifo_size,
3877                                         pixel_size, latency->cursor_hpll_disable);
3878                 reg = I915_READ(DSPFW3);
3879                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
3880                 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
3881                 I915_WRITE(DSPFW3, reg);
3882                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
3883
3884                 /* activate cxsr */
3885                 I915_WRITE(DSPFW3,
3886                            I915_READ(DSPFW3) | PINEVIEW_SELF_REFRESH_EN);
3887                 DRM_DEBUG_KMS("Self-refresh is enabled\n");
3888         } else {
3889                 pineview_disable_cxsr(dev);
3890                 DRM_DEBUG_KMS("Self-refresh is disabled\n");
3891         }
3892 }
3893
3894 static bool g4x_compute_wm0(struct drm_device *dev,
3895                             int plane,
3896                             const struct intel_watermark_params *display,
3897                             int display_latency_ns,
3898                             const struct intel_watermark_params *cursor,
3899                             int cursor_latency_ns,
3900                             int *plane_wm,
3901                             int *cursor_wm)
3902 {
3903         struct drm_crtc *crtc;
3904         int htotal, hdisplay, clock, pixel_size;
3905         int line_time_us, line_count;
3906         int entries, tlb_miss;
3907
3908         crtc = intel_get_crtc_for_plane(dev, plane);
3909         if (crtc->fb == NULL || !crtc->enabled) {
3910                 *cursor_wm = cursor->guard_size;
3911                 *plane_wm = display->guard_size;
3912                 return false;
3913         }
3914
3915         htotal = crtc->mode.htotal;
3916         hdisplay = crtc->mode.hdisplay;
3917         clock = crtc->mode.clock;
3918         pixel_size = crtc->fb->bits_per_pixel / 8;
3919
3920         /* Use the small buffer method to calculate plane watermark */
3921         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
3922         tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
3923         if (tlb_miss > 0)
3924                 entries += tlb_miss;
3925         entries = DIV_ROUND_UP(entries, display->cacheline_size);
3926         *plane_wm = entries + display->guard_size;
3927         if (*plane_wm > (int)display->max_wm)
3928                 *plane_wm = display->max_wm;
3929
3930         /* Use the large buffer method to calculate cursor watermark */
3931         line_time_us = ((htotal * 1000) / clock);
3932         line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
3933         entries = line_count * 64 * pixel_size;
3934         tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
3935         if (tlb_miss > 0)
3936                 entries += tlb_miss;
3937         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
3938         *cursor_wm = entries + cursor->guard_size;
3939         if (*cursor_wm > (int)cursor->max_wm)
3940                 *cursor_wm = (int)cursor->max_wm;
3941
3942         return true;
3943 }
3944
3945 /*
3946  * Check the wm result.
3947  *
3948  * If any calculated watermark values is larger than the maximum value that
3949  * can be programmed into the associated watermark register, that watermark
3950  * must be disabled.
3951  */
3952 static bool g4x_check_srwm(struct drm_device *dev,
3953                            int display_wm, int cursor_wm,
3954                            const struct intel_watermark_params *display,
3955                            const struct intel_watermark_params *cursor)
3956 {
3957         DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
3958                       display_wm, cursor_wm);
3959
3960         if (display_wm > display->max_wm) {
3961                 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
3962                               display_wm, display->max_wm);
3963                 return false;
3964         }
3965
3966         if (cursor_wm > cursor->max_wm) {
3967                 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
3968                               cursor_wm, cursor->max_wm);
3969                 return false;
3970         }
3971
3972         if (!(display_wm || cursor_wm)) {
3973                 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
3974                 return false;
3975         }
3976
3977         return true;
3978 }
3979
3980 static bool g4x_compute_srwm(struct drm_device *dev,
3981                              int plane,
3982                              int latency_ns,
3983                              const struct intel_watermark_params *display,
3984                              const struct intel_watermark_params *cursor,
3985                              int *display_wm, int *cursor_wm)
3986 {
3987         struct drm_crtc *crtc;
3988         int hdisplay, htotal, pixel_size, clock;
3989         unsigned long line_time_us;
3990         int line_count, line_size;
3991         int small, large;
3992         int entries;
3993
3994         if (!latency_ns) {
3995                 *display_wm = *cursor_wm = 0;
3996                 return false;
3997         }
3998
3999         crtc = intel_get_crtc_for_plane(dev, plane);
4000         hdisplay = crtc->mode.hdisplay;
4001         htotal = crtc->mode.htotal;
4002         clock = crtc->mode.clock;
4003         pixel_size = crtc->fb->bits_per_pixel / 8;
4004
4005         line_time_us = (htotal * 1000) / clock;
4006         line_count = (latency_ns / line_time_us + 1000) / 1000;
4007         line_size = hdisplay * pixel_size;
4008
4009         /* Use the minimum of the small and large buffer method for primary */
4010         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
4011         large = line_count * line_size;
4012
4013         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
4014         *display_wm = entries + display->guard_size;
4015
4016         /* calculate the self-refresh watermark for display cursor */
4017         entries = line_count * pixel_size * 64;
4018         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
4019         *cursor_wm = entries + cursor->guard_size;
4020
4021         return g4x_check_srwm(dev,
4022                               *display_wm, *cursor_wm,
4023                               display, cursor);
4024 }
4025
4026 #define single_plane_enabled(mask) is_power_of_2(mask)
4027
4028 static void g4x_update_wm(struct drm_device *dev)
4029 {
4030         static const int sr_latency_ns = 12000;
4031         struct drm_i915_private *dev_priv = dev->dev_private;
4032         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
4033         int plane_sr, cursor_sr;
4034         unsigned int enabled = 0;
4035
4036         if (g4x_compute_wm0(dev, 0,
4037                             &g4x_wm_info, latency_ns,
4038                             &g4x_cursor_wm_info, latency_ns,
4039                             &planea_wm, &cursora_wm))
4040                 enabled |= 1;
4041
4042         if (g4x_compute_wm0(dev, 1,
4043                             &g4x_wm_info, latency_ns,
4044                             &g4x_cursor_wm_info, latency_ns,
4045                             &planeb_wm, &cursorb_wm))
4046                 enabled |= 2;
4047
4048         plane_sr = cursor_sr = 0;
4049         if (single_plane_enabled(enabled) &&
4050             g4x_compute_srwm(dev, ffs(enabled) - 1,
4051                              sr_latency_ns,
4052                              &g4x_wm_info,
4053                              &g4x_cursor_wm_info,
4054                              &plane_sr, &cursor_sr))
4055                 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
4056         else
4057                 I915_WRITE(FW_BLC_SELF,
4058                            I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
4059
4060         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
4061                       planea_wm, cursora_wm,
4062                       planeb_wm, cursorb_wm,
4063                       plane_sr, cursor_sr);
4064
4065         I915_WRITE(DSPFW1,
4066                    (plane_sr << DSPFW_SR_SHIFT) |
4067                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
4068                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
4069                    planea_wm);
4070         I915_WRITE(DSPFW2,
4071                    (I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) |
4072                    (cursora_wm << DSPFW_CURSORA_SHIFT));
4073         /* HPLL off in SR has some issues on G4x... disable it */
4074         I915_WRITE(DSPFW3,
4075                    (I915_READ(DSPFW3) & ~DSPFW_HPLL_SR_EN) |
4076                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
4077 }
4078
4079 static void i965_update_wm(struct drm_device *dev)
4080 {
4081         struct drm_i915_private *dev_priv = dev->dev_private;
4082         struct drm_crtc *crtc;
4083         int srwm = 1;
4084         int cursor_sr = 16;
4085
4086         /* Calc sr entries for one plane configs */
4087         crtc = single_enabled_crtc(dev);
4088         if (crtc) {
4089                 /* self-refresh has much higher latency */
4090                 static const int sr_latency_ns = 12000;
4091                 int clock = crtc->mode.clock;
4092                 int htotal = crtc->mode.htotal;
4093                 int hdisplay = crtc->mode.hdisplay;
4094                 int pixel_size = crtc->fb->bits_per_pixel / 8;
4095                 unsigned long line_time_us;
4096                 int entries;
4097
4098                 line_time_us = ((htotal * 1000) / clock);
4099
4100                 /* Use ns/us then divide to preserve precision */
4101                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
4102                         pixel_size * hdisplay;
4103                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
4104                 srwm = I965_FIFO_SIZE - entries;
4105                 if (srwm < 0)
4106                         srwm = 1;
4107                 srwm &= 0x1ff;
4108                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
4109                               entries, srwm);
4110
4111                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
4112                         pixel_size * 64;
4113                 entries = DIV_ROUND_UP(entries,
4114                                           i965_cursor_wm_info.cacheline_size);
4115                 cursor_sr = i965_cursor_wm_info.fifo_size -
4116                         (entries + i965_cursor_wm_info.guard_size);
4117
4118                 if (cursor_sr > i965_cursor_wm_info.max_wm)
4119                         cursor_sr = i965_cursor_wm_info.max_wm;
4120
4121                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
4122                               "cursor %d\n", srwm, cursor_sr);
4123
4124                 if (IS_CRESTLINE(dev))
4125                         I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
4126         } else {
4127                 /* Turn off self refresh if both pipes are enabled */
4128                 if (IS_CRESTLINE(dev))
4129                         I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
4130                                    & ~FW_BLC_SELF_EN);
4131         }
4132
4133         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
4134                       srwm);
4135
4136         /* 965 has limitations... */
4137         I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
4138                    (8 << 16) | (8 << 8) | (8 << 0));
4139         I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
4140         /* update cursor SR watermark */
4141         I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
4142 }
4143
4144 static void i9xx_update_wm(struct drm_device *dev)
4145 {
4146         struct drm_i915_private *dev_priv = dev->dev_private;
4147         const struct intel_watermark_params *wm_info;
4148         uint32_t fwater_lo;
4149         uint32_t fwater_hi;
4150         int cwm, srwm = 1;
4151         int fifo_size;
4152         int planea_wm, planeb_wm;
4153         struct drm_crtc *crtc, *enabled = NULL;
4154
4155         if (IS_I945GM(dev))
4156                 wm_info = &i945_wm_info;
4157         else if (!IS_GEN2(dev))
4158                 wm_info = &i915_wm_info;
4159         else
4160                 wm_info = &i855_wm_info;
4161
4162         fifo_size = dev_priv->display.get_fifo_size(dev, 0);
4163         crtc = intel_get_crtc_for_plane(dev, 0);
4164         if (crtc->enabled && crtc->fb) {
4165                 planea_wm = intel_calculate_wm(crtc->mode.clock,
4166                                                wm_info, fifo_size,
4167                                                crtc->fb->bits_per_pixel / 8,
4168                                                latency_ns);
4169                 enabled = crtc;
4170         } else
4171                 planea_wm = fifo_size - wm_info->guard_size;
4172
4173         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
4174         crtc = intel_get_crtc_for_plane(dev, 1);
4175         if (crtc->enabled && crtc->fb) {
4176                 planeb_wm = intel_calculate_wm(crtc->mode.clock,
4177                                                wm_info, fifo_size,
4178                                                crtc->fb->bits_per_pixel / 8,
4179                                                latency_ns);
4180                 if (enabled == NULL)
4181                         enabled = crtc;
4182                 else
4183                         enabled = NULL;
4184         } else
4185                 planeb_wm = fifo_size - wm_info->guard_size;
4186
4187         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
4188
4189         /*
4190          * Overlay gets an aggressive default since video jitter is bad.
4191          */
4192         cwm = 2;
4193
4194         /* Play safe and disable self-refresh before adjusting watermarks. */
4195         if (IS_I945G(dev) || IS_I945GM(dev))
4196                 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
4197         else if (IS_I915GM(dev))
4198                 I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
4199
4200         /* Calc sr entries for one plane configs */
4201         if (HAS_FW_BLC(dev) && enabled) {
4202                 /* self-refresh has much higher latency */
4203                 static const int sr_latency_ns = 6000;
4204                 int clock = enabled->mode.clock;
4205                 int htotal = enabled->mode.htotal;
4206                 int hdisplay = enabled->mode.hdisplay;
4207                 int pixel_size = enabled->fb->bits_per_pixel / 8;
4208                 unsigned long line_time_us;
4209                 int entries;
4210
4211                 line_time_us = (htotal * 1000) / clock;
4212
4213                 /* Use ns/us then divide to preserve precision */
4214                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
4215                         pixel_size * hdisplay;
4216                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
4217                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
4218                 srwm = wm_info->fifo_size - entries;
4219                 if (srwm < 0)
4220                         srwm = 1;
4221
4222                 if (IS_I945G(dev) || IS_I945GM(dev))
4223                         I915_WRITE(FW_BLC_SELF,
4224                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
4225                 else if (IS_I915GM(dev))
4226                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
4227         }
4228
4229         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
4230                       planea_wm, planeb_wm, cwm, srwm);
4231
4232         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
4233         fwater_hi = (cwm & 0x1f);
4234
4235         /* Set request length to 8 cachelines per fetch */
4236         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
4237         fwater_hi = fwater_hi | (1 << 8);
4238
4239         I915_WRITE(FW_BLC, fwater_lo);
4240         I915_WRITE(FW_BLC2, fwater_hi);
4241
4242         if (HAS_FW_BLC(dev)) {
4243                 if (enabled) {
4244                         if (IS_I945G(dev) || IS_I945GM(dev))
4245                                 I915_WRITE(FW_BLC_SELF,
4246                                            FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
4247                         else if (IS_I915GM(dev))
4248                                 I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN);
4249                         DRM_DEBUG_KMS("memory self refresh enabled\n");
4250                 } else
4251                         DRM_DEBUG_KMS("memory self refresh disabled\n");
4252         }
4253 }
4254
4255 static void i830_update_wm(struct drm_device *dev)
4256 {
4257         struct drm_i915_private *dev_priv = dev->dev_private;
4258         struct drm_crtc *crtc;
4259         uint32_t fwater_lo;
4260         int planea_wm;
4261
4262         crtc = single_enabled_crtc(dev);
4263         if (crtc == NULL)
4264                 return;
4265
4266         planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
4267                                        dev_priv->display.get_fifo_size(dev, 0),
4268                                        crtc->fb->bits_per_pixel / 8,
4269                                        latency_ns);
4270         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
4271         fwater_lo |= (3<<8) | planea_wm;
4272
4273         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
4274
4275         I915_WRITE(FW_BLC, fwater_lo);
4276 }
4277
4278 #define ILK_LP0_PLANE_LATENCY           700
4279 #define ILK_LP0_CURSOR_LATENCY          1300
4280
4281 /*
4282  * Check the wm result.
4283  *
4284  * If any calculated watermark values is larger than the maximum value that
4285  * can be programmed into the associated watermark register, that watermark
4286  * must be disabled.
4287  */
4288 static bool ironlake_check_srwm(struct drm_device *dev, int level,
4289                                 int fbc_wm, int display_wm, int cursor_wm,
4290                                 const struct intel_watermark_params *display,
4291                                 const struct intel_watermark_params *cursor)
4292 {
4293         struct drm_i915_private *dev_priv = dev->dev_private;
4294
4295         DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
4296                       " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
4297
4298         if (fbc_wm > SNB_FBC_MAX_SRWM) {
4299                 DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
4300                               fbc_wm, SNB_FBC_MAX_SRWM, level);
4301
4302                 /* fbc has it's own way to disable FBC WM */
4303                 I915_WRITE(DISP_ARB_CTL,
4304                            I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
4305                 return false;
4306         }
4307
4308         if (display_wm > display->max_wm) {
4309                 DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
4310                               display_wm, SNB_DISPLAY_MAX_SRWM, level);
4311                 return false;
4312         }
4313
4314         if (cursor_wm > cursor->max_wm) {
4315                 DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
4316                               cursor_wm, SNB_CURSOR_MAX_SRWM, level);
4317                 return false;
4318         }
4319
4320         if (!(fbc_wm || display_wm || cursor_wm)) {
4321                 DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
4322                 return false;
4323         }
4324
4325         return true;
4326 }
4327
4328 /*
4329  * Compute watermark values of WM[1-3],
4330  */
4331 static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
4332                                   int latency_ns,
4333                                   const struct intel_watermark_params *display,
4334                                   const struct intel_watermark_params *cursor,
4335                                   int *fbc_wm, int *display_wm, int *cursor_wm)
4336 {
4337         struct drm_crtc *crtc;
4338         unsigned long line_time_us;
4339         int hdisplay, htotal, pixel_size, clock;
4340         int line_count, line_size;
4341         int small, large;
4342         int entries;
4343
4344         if (!latency_ns) {
4345                 *fbc_wm = *display_wm = *cursor_wm = 0;
4346                 return false;
4347         }
4348
4349         crtc = intel_get_crtc_for_plane(dev, plane);
4350         hdisplay = crtc->mode.hdisplay;
4351         htotal = crtc->mode.htotal;
4352         clock = crtc->mode.clock;
4353         pixel_size = crtc->fb->bits_per_pixel / 8;
4354
4355         line_time_us = (htotal * 1000) / clock;
4356         line_count = (latency_ns / line_time_us + 1000) / 1000;
4357         line_size = hdisplay * pixel_size;
4358
4359         /* Use the minimum of the small and large buffer method for primary */
4360         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
4361         large = line_count * line_size;
4362
4363         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
4364         *display_wm = entries + display->guard_size;
4365
4366         /*
4367          * Spec says:
4368          * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
4369          */
4370         *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
4371
4372         /* calculate the self-refresh watermark for display cursor */
4373         entries = line_count * pixel_size * 64;
4374         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
4375         *cursor_wm = entries + cursor->guard_size;
4376
4377         return ironlake_check_srwm(dev, level,
4378                                    *fbc_wm, *display_wm, *cursor_wm,
4379                                    display, cursor);
4380 }
4381
4382 static void ironlake_update_wm(struct drm_device *dev)
4383 {
4384         struct drm_i915_private *dev_priv = dev->dev_private;
4385         int fbc_wm, plane_wm, cursor_wm;
4386         unsigned int enabled;
4387
4388         enabled = 0;
4389         if (g4x_compute_wm0(dev, 0,
4390                             &ironlake_display_wm_info,
4391                             ILK_LP0_PLANE_LATENCY,
4392                             &ironlake_cursor_wm_info,
4393                             ILK_LP0_CURSOR_LATENCY,
4394                             &plane_wm, &cursor_wm)) {
4395                 I915_WRITE(WM0_PIPEA_ILK,
4396                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
4397                 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
4398                               " plane %d, " "cursor: %d\n",
4399                               plane_wm, cursor_wm);
4400                 enabled |= 1;
4401         }
4402
4403         if (g4x_compute_wm0(dev, 1,
4404                             &ironlake_display_wm_info,
4405                             ILK_LP0_PLANE_LATENCY,
4406                             &ironlake_cursor_wm_info,
4407                             ILK_LP0_CURSOR_LATENCY,
4408                             &plane_wm, &cursor_wm)) {
4409                 I915_WRITE(WM0_PIPEB_ILK,
4410                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
4411                 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
4412                               " plane %d, cursor: %d\n",
4413                               plane_wm, cursor_wm);
4414                 enabled |= 2;
4415         }
4416
4417         /*
4418          * Calculate and update the self-refresh watermark only when one
4419          * display plane is used.
4420          */
4421         I915_WRITE(WM3_LP_ILK, 0);
4422         I915_WRITE(WM2_LP_ILK, 0);
4423         I915_WRITE(WM1_LP_ILK, 0);
4424
4425         if (!single_plane_enabled(enabled))
4426                 return;
4427         enabled = ffs(enabled) - 1;
4428
4429         /* WM1 */
4430         if (!ironlake_compute_srwm(dev, 1, enabled,
4431                                    ILK_READ_WM1_LATENCY() * 500,
4432                                    &ironlake_display_srwm_info,
4433                                    &ironlake_cursor_srwm_info,
4434                                    &fbc_wm, &plane_wm, &cursor_wm))
4435                 return;
4436
4437         I915_WRITE(WM1_LP_ILK,
4438                    WM1_LP_SR_EN |
4439                    (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4440                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4441                    (plane_wm << WM1_LP_SR_SHIFT) |
4442                    cursor_wm);
4443
4444         /* WM2 */
4445         if (!ironlake_compute_srwm(dev, 2, enabled,
4446                                    ILK_READ_WM2_LATENCY() * 500,
4447                                    &ironlake_display_srwm_info,
4448                                    &ironlake_cursor_srwm_info,
4449                                    &fbc_wm, &plane_wm, &cursor_wm))
4450                 return;
4451
4452         I915_WRITE(WM2_LP_ILK,
4453                    WM2_LP_EN |
4454                    (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4455                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4456                    (plane_wm << WM1_LP_SR_SHIFT) |
4457                    cursor_wm);
4458
4459         /*
4460          * WM3 is unsupported on ILK, probably because we don't have latency
4461          * data for that power state
4462          */
4463 }
4464
4465 static void sandybridge_update_wm(struct drm_device *dev)
4466 {
4467         struct drm_i915_private *dev_priv = dev->dev_private;
4468         int latency = SNB_READ_WM0_LATENCY() * 100;     /* In unit 0.1us */
4469         int fbc_wm, plane_wm, cursor_wm;
4470         unsigned int enabled;
4471
4472         enabled = 0;
4473         if (g4x_compute_wm0(dev, 0,
4474                             &sandybridge_display_wm_info, latency,
4475                             &sandybridge_cursor_wm_info, latency,
4476                             &plane_wm, &cursor_wm)) {
4477                 I915_WRITE(WM0_PIPEA_ILK,
4478                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
4479                 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
4480                               " plane %d, " "cursor: %d\n",
4481                               plane_wm, cursor_wm);
4482                 enabled |= 1;
4483         }
4484
4485         if (g4x_compute_wm0(dev, 1,
4486                             &sandybridge_display_wm_info, latency,
4487                             &sandybridge_cursor_wm_info, latency,
4488                             &plane_wm, &cursor_wm)) {
4489                 I915_WRITE(WM0_PIPEB_ILK,
4490                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
4491                 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
4492                               " plane %d, cursor: %d\n",
4493                               plane_wm, cursor_wm);
4494                 enabled |= 2;
4495         }
4496
4497         /*
4498          * Calculate and update the self-refresh watermark only when one
4499          * display plane is used.
4500          *
4501          * SNB support 3 levels of watermark.
4502          *
4503          * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
4504          * and disabled in the descending order
4505          *
4506          */
4507         I915_WRITE(WM3_LP_ILK, 0);
4508         I915_WRITE(WM2_LP_ILK, 0);
4509         I915_WRITE(WM1_LP_ILK, 0);
4510
4511         if (!single_plane_enabled(enabled))
4512                 return;
4513         enabled = ffs(enabled) - 1;
4514
4515         /* WM1 */
4516         if (!ironlake_compute_srwm(dev, 1, enabled,
4517                                    SNB_READ_WM1_LATENCY() * 500,
4518                                    &sandybridge_display_srwm_info,
4519                                    &sandybridge_cursor_srwm_info,
4520                                    &fbc_wm, &plane_wm, &cursor_wm))
4521                 return;
4522
4523         I915_WRITE(WM1_LP_ILK,
4524                    WM1_LP_SR_EN |
4525                    (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4526                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4527                    (plane_wm << WM1_LP_SR_SHIFT) |
4528                    cursor_wm);
4529
4530         /* WM2 */
4531         if (!ironlake_compute_srwm(dev, 2, enabled,
4532                                    SNB_READ_WM2_LATENCY() * 500,
4533                                    &sandybridge_display_srwm_info,
4534                                    &sandybridge_cursor_srwm_info,
4535                                    &fbc_wm, &plane_wm, &cursor_wm))
4536                 return;
4537
4538         I915_WRITE(WM2_LP_ILK,
4539                    WM2_LP_EN |
4540                    (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4541                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4542                    (plane_wm << WM1_LP_SR_SHIFT) |
4543                    cursor_wm);
4544
4545         /* WM3 */
4546         if (!ironlake_compute_srwm(dev, 3, enabled,
4547                                    SNB_READ_WM3_LATENCY() * 500,
4548                                    &sandybridge_display_srwm_info,
4549                                    &sandybridge_cursor_srwm_info,
4550                                    &fbc_wm, &plane_wm, &cursor_wm))
4551                 return;
4552
4553         I915_WRITE(WM3_LP_ILK,
4554                    WM3_LP_EN |
4555                    (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4556                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4557                    (plane_wm << WM1_LP_SR_SHIFT) |
4558                    cursor_wm);
4559 }
4560
4561 /**
4562  * intel_update_watermarks - update FIFO watermark values based on current modes
4563  *
4564  * Calculate watermark values for the various WM regs based on current mode
4565  * and plane configuration.
4566  *
4567  * There are several cases to deal with here:
4568  *   - normal (i.e. non-self-refresh)
4569  *   - self-refresh (SR) mode
4570  *   - lines are large relative to FIFO size (buffer can hold up to 2)
4571  *   - lines are small relative to FIFO size (buffer can hold more than 2
4572  *     lines), so need to account for TLB latency
4573  *
4574  *   The normal calculation is:
4575  *     watermark = dotclock * bytes per pixel * latency
4576  *   where latency is platform & configuration dependent (we assume pessimal
4577  *   values here).
4578  *
4579  *   The SR calculation is:
4580  *     watermark = (trunc(latency/line time)+1) * surface width *
4581  *       bytes per pixel
4582  *   where
4583  *     line time = htotal / dotclock
4584  *     surface width = hdisplay for normal plane and 64 for cursor
4585  *   and latency is assumed to be high, as above.
4586  *
4587  * The final value programmed to the register should always be rounded up,
4588  * and include an extra 2 entries to account for clock crossings.
4589  *
4590  * We don't use the sprite, so we can ignore that.  And on Crestline we have
4591  * to set the non-SR watermarks to 8.
4592  */
4593 static void intel_update_watermarks(struct drm_device *dev)
4594 {
4595         struct drm_i915_private *dev_priv = dev->dev_private;
4596
4597         if (dev_priv->display.update_wm)
4598                 dev_priv->display.update_wm(dev);
4599 }
4600
4601 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
4602 {
4603         if (i915_panel_use_ssc >= 0)
4604                 return i915_panel_use_ssc != 0;
4605         return dev_priv->lvds_use_ssc
4606                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
4607 }
4608
4609 /**
4610  * intel_choose_pipe_bpp_dither - figure out what color depth the pipe should send
4611  * @crtc: CRTC structure
4612  *
4613  * A pipe may be connected to one or more outputs.  Based on the depth of the
4614  * attached framebuffer, choose a good color depth to use on the pipe.
4615  *
4616  * If possible, match the pipe depth to the fb depth.  In some cases, this
4617  * isn't ideal, because the connected output supports a lesser or restricted
4618  * set of depths.  Resolve that here:
4619  *    LVDS typically supports only 6bpc, so clamp down in that case
4620  *    HDMI supports only 8bpc or 12bpc, so clamp to 8bpc with dither for 10bpc
4621  *    Displays may support a restricted set as well, check EDID and clamp as
4622  *      appropriate.
4623  *
4624  * RETURNS:
4625  * Dithering requirement (i.e. false if display bpc and pipe bpc match,
4626  * true if they don't match).
4627  */
4628 static bool intel_choose_pipe_bpp_dither(struct drm_crtc *crtc,
4629                                          unsigned int *pipe_bpp)
4630 {
4631         struct drm_device *dev = crtc->dev;
4632         struct drm_i915_private *dev_priv = dev->dev_private;
4633         struct drm_encoder *encoder;
4634         struct drm_connector *connector;
4635         unsigned int display_bpc = UINT_MAX, bpc;
4636
4637         /* Walk the encoders & connectors on this crtc, get min bpc */
4638         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
4639                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
4640
4641                 if (encoder->crtc != crtc)
4642                         continue;
4643
4644                 if (intel_encoder->type == INTEL_OUTPUT_LVDS) {
4645                         unsigned int lvds_bpc;
4646
4647                         if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) ==
4648                             LVDS_A3_POWER_UP)
4649                                 lvds_bpc = 8;
4650                         else
4651                                 lvds_bpc = 6;
4652
4653                         if (lvds_bpc < display_bpc) {
4654                                 DRM_DEBUG_DRIVER("clamping display bpc (was %d) to LVDS (%d)\n", display_bpc, lvds_bpc);
4655                                 display_bpc = lvds_bpc;
4656                         }
4657                         continue;
4658                 }
4659
4660                 if (intel_encoder->type == INTEL_OUTPUT_EDP) {
4661                         /* Use VBT settings if we have an eDP panel */
4662                         unsigned int edp_bpc = dev_priv->edp.bpp / 3;
4663
4664                         if (edp_bpc < display_bpc) {
4665                                 DRM_DEBUG_DRIVER("clamping display bpc (was %d) to eDP (%d)\n", display_bpc, edp_bpc);
4666                                 display_bpc = edp_bpc;
4667                         }
4668                         continue;
4669                 }
4670
4671                 /* Not one of the known troublemakers, check the EDID */
4672                 list_for_each_entry(connector, &dev->mode_config.connector_list,
4673                                     head) {
4674                         if (connector->encoder != encoder)
4675                                 continue;
4676
4677                         /* Don't use an invalid EDID bpc value */
4678                         if (connector->display_info.bpc &&
4679                             connector->display_info.bpc < display_bpc) {
4680                                 DRM_DEBUG_DRIVER("clamping display bpc (was %d) to EDID reported max of %d\n", display_bpc, connector->display_info.bpc);
4681                                 display_bpc = connector->display_info.bpc;
4682                         }
4683                 }
4684
4685                 /*
4686                  * HDMI is either 12 or 8, so if the display lets 10bpc sneak
4687                  * through, clamp it down.  (Note: >12bpc will be caught below.)
4688                  */
4689                 if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
4690                         if (display_bpc > 8 && display_bpc < 12) {
4691                                 DRM_DEBUG_DRIVER("forcing bpc to 12 for HDMI\n");
4692                                 display_bpc = 12;
4693                         } else {
4694                                 DRM_DEBUG_DRIVER("forcing bpc to 8 for HDMI\n");
4695                                 display_bpc = 8;
4696                         }
4697                 }
4698         }
4699
4700         /*
4701          * We could just drive the pipe at the highest bpc all the time and
4702          * enable dithering as needed, but that costs bandwidth.  So choose
4703          * the minimum value that expresses the full color range of the fb but
4704          * also stays within the max display bpc discovered above.
4705          */
4706
4707         switch (crtc->fb->depth) {
4708         case 8:
4709                 bpc = 8; /* since we go through a colormap */
4710                 break;
4711         case 15:
4712         case 16:
4713                 bpc = 6; /* min is 18bpp */
4714                 break;
4715         case 24:
4716                 bpc = 8;
4717                 break;
4718         case 30:
4719                 bpc = 10;
4720                 break;
4721         case 48:
4722                 bpc = 12;
4723                 break;
4724         default:
4725                 DRM_DEBUG("unsupported depth, assuming 24 bits\n");
4726                 bpc = min((unsigned int)8, display_bpc);
4727                 break;
4728         }
4729
4730         display_bpc = min(display_bpc, bpc);
4731
4732         DRM_DEBUG_DRIVER("setting pipe bpc to %d (max display bpc %d)\n",
4733                          bpc, display_bpc);
4734
4735         *pipe_bpp = display_bpc * 3;
4736
4737         return display_bpc != bpc;
4738 }
4739
4740 static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
4741                               struct drm_display_mode *mode,
4742                               struct drm_display_mode *adjusted_mode,
4743                               int x, int y,
4744                               struct drm_framebuffer *old_fb)
4745 {
4746         struct drm_device *dev = crtc->dev;
4747         struct drm_i915_private *dev_priv = dev->dev_private;
4748         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4749         int pipe = intel_crtc->pipe;
4750         int plane = intel_crtc->plane;
4751         int refclk, num_connectors = 0;
4752         intel_clock_t clock, reduced_clock;
4753         u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf;
4754         bool ok, has_reduced_clock = false, is_sdvo = false, is_dvo = false;
4755         bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
4756         struct drm_mode_config *mode_config = &dev->mode_config;
4757         struct intel_encoder *encoder;
4758         const intel_limit_t *limit;
4759         int ret;
4760         u32 temp;
4761         u32 lvds_sync = 0;
4762
4763         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
4764                 if (encoder->base.crtc != crtc)
4765                         continue;
4766
4767                 switch (encoder->type) {
4768                 case INTEL_OUTPUT_LVDS:
4769                         is_lvds = true;
4770                         break;
4771                 case INTEL_OUTPUT_SDVO:
4772                 case INTEL_OUTPUT_HDMI:
4773                         is_sdvo = true;
4774                         if (encoder->needs_tv_clock)
4775                                 is_tv = true;
4776                         break;
4777                 case INTEL_OUTPUT_DVO:
4778                         is_dvo = true;
4779                         break;
4780                 case INTEL_OUTPUT_TVOUT:
4781                         is_tv = true;
4782                         break;
4783                 case INTEL_OUTPUT_ANALOG:
4784                         is_crt = true;
4785                         break;
4786                 case INTEL_OUTPUT_DISPLAYPORT:
4787                         is_dp = true;
4788                         break;
4789                 }
4790
4791                 num_connectors++;
4792         }
4793
4794         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
4795                 refclk = dev_priv->lvds_ssc_freq * 1000;
4796                 DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
4797                               refclk / 1000);
4798         } else if (!IS_GEN2(dev)) {
4799                 refclk = 96000;
4800         } else {
4801                 refclk = 48000;
4802         }
4803
4804         /*
4805          * Returns a set of divisors for the desired target clock with the given
4806          * refclk, or FALSE.  The returned values represent the clock equation:
4807          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
4808          */
4809         limit = intel_limit(crtc, refclk);
4810         ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, &clock);
4811         if (!ok) {
4812                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
4813                 return -EINVAL;
4814         }
4815
4816         /* Ensure that the cursor is valid for the new mode before changing... */
4817         intel_crtc_update_cursor(crtc, true);
4818
4819         if (is_lvds && dev_priv->lvds_downclock_avail) {
4820                 has_reduced_clock = limit->find_pll(limit, crtc,
4821                                                     dev_priv->lvds_downclock,
4822                                                     refclk,
4823                                                     &reduced_clock);
4824                 if (has_reduced_clock && (clock.p != reduced_clock.p)) {
4825                         /*
4826                          * If the different P is found, it means that we can't
4827                          * switch the display clock by using the FP0/FP1.
4828                          * In such case we will disable the LVDS downclock
4829                          * feature.
4830                          */
4831                         DRM_DEBUG_KMS("Different P is found for "
4832                                       "LVDS clock/downclock\n");
4833                         has_reduced_clock = 0;
4834                 }
4835         }
4836         /* SDVO TV has fixed PLL values depend on its clock range,
4837            this mirrors vbios setting. */
4838         if (is_sdvo && is_tv) {
4839                 if (adjusted_mode->clock >= 100000
4840                     && adjusted_mode->clock < 140500) {
4841                         clock.p1 = 2;
4842                         clock.p2 = 10;
4843                         clock.n = 3;
4844                         clock.m1 = 16;
4845                         clock.m2 = 8;
4846                 } else if (adjusted_mode->clock >= 140500
4847                            && adjusted_mode->clock <= 200000) {
4848                         clock.p1 = 1;
4849                         clock.p2 = 10;
4850                         clock.n = 6;
4851                         clock.m1 = 12;
4852                         clock.m2 = 8;
4853                 }
4854         }
4855
4856         if (IS_PINEVIEW(dev)) {
4857                 fp = (1 << clock.n) << 16 | clock.m1 << 8 | clock.m2;
4858                 if (has_reduced_clock)
4859                         fp2 = (1 << reduced_clock.n) << 16 |
4860                                 reduced_clock.m1 << 8 | reduced_clock.m2;
4861         } else {
4862                 fp = clock.n << 16 | clock.m1 << 8 | clock.m2;
4863                 if (has_reduced_clock)
4864                         fp2 = reduced_clock.n << 16 | reduced_clock.m1 << 8 |
4865                                 reduced_clock.m2;
4866         }
4867
4868         dpll = DPLL_VGA_MODE_DIS;
4869
4870         if (!IS_GEN2(dev)) {
4871                 if (is_lvds)
4872                         dpll |= DPLLB_MODE_LVDS;
4873                 else
4874                         dpll |= DPLLB_MODE_DAC_SERIAL;
4875                 if (is_sdvo) {
4876                         int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
4877                         if (pixel_multiplier > 1) {
4878                                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4879                                         dpll |= (pixel_multiplier - 1) << SDVO_MULTIPLIER_SHIFT_HIRES;
4880                         }
4881                         dpll |= DPLL_DVO_HIGH_SPEED;
4882                 }
4883                 if (is_dp)
4884                         dpll |= DPLL_DVO_HIGH_SPEED;
4885
4886                 /* compute bitmask from p1 value */
4887                 if (IS_PINEVIEW(dev))
4888                         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
4889                 else {
4890                         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
4891                         if (IS_G4X(dev) && has_reduced_clock)
4892                                 dpll |= (1 << (reduced_clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
4893                 }
4894                 switch (clock.p2) {
4895                 case 5:
4896                         dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
4897                         break;
4898                 case 7:
4899                         dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
4900                         break;
4901                 case 10:
4902                         dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
4903                         break;
4904                 case 14:
4905                         dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
4906                         break;
4907                 }
4908                 if (INTEL_INFO(dev)->gen >= 4)
4909                         dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
4910         } else {
4911                 if (is_lvds) {
4912                         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
4913                 } else {
4914                         if (clock.p1 == 2)
4915                                 dpll |= PLL_P1_DIVIDE_BY_TWO;
4916                         else
4917                                 dpll |= (clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
4918                         if (clock.p2 == 4)
4919                                 dpll |= PLL_P2_DIVIDE_BY_4;
4920                 }
4921         }
4922
4923         if (is_sdvo && is_tv)
4924                 dpll |= PLL_REF_INPUT_TVCLKINBC;
4925         else if (is_tv)
4926                 /* XXX: just matching BIOS for now */
4927                 /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
4928                 dpll |= 3;
4929         else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
4930                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
4931         else
4932                 dpll |= PLL_REF_INPUT_DREFCLK;
4933
4934         /* setup pipeconf */
4935         pipeconf = I915_READ(PIPECONF(pipe));
4936
4937         /* Set up the display plane register */
4938         dspcntr = DISPPLANE_GAMMA_ENABLE;
4939
4940         /* Ironlake's plane is forced to pipe, bit 24 is to
4941            enable color space conversion */
4942         if (pipe == 0)
4943                 dspcntr &= ~DISPPLANE_SEL_PIPE_MASK;
4944         else
4945                 dspcntr |= DISPPLANE_SEL_PIPE_B;
4946
4947         if (pipe == 0 && INTEL_INFO(dev)->gen < 4) {
4948                 /* Enable pixel doubling when the dot clock is > 90% of the (display)
4949                  * core speed.
4950                  *
4951                  * XXX: No double-wide on 915GM pipe B. Is that the only reason for the
4952                  * pipe == 0 check?
4953                  */
4954                 if (mode->clock >
4955                     dev_priv->display.get_display_clock_speed(dev) * 9 / 10)
4956                         pipeconf |= PIPECONF_DOUBLE_WIDE;
4957                 else
4958                         pipeconf &= ~PIPECONF_DOUBLE_WIDE;
4959         }
4960
4961         dpll |= DPLL_VCO_ENABLE;
4962
4963         DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B');
4964         drm_mode_debug_printmodeline(mode);
4965
4966         I915_WRITE(FP0(pipe), fp);
4967         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
4968
4969         POSTING_READ(DPLL(pipe));
4970         udelay(150);
4971
4972         /* The LVDS pin pair needs to be on before the DPLLs are enabled.
4973          * This is an exception to the general rule that mode_set doesn't turn
4974          * things on.
4975          */
4976         if (is_lvds) {
4977                 temp = I915_READ(LVDS);
4978                 temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
4979                 if (pipe == 1) {
4980                         temp |= LVDS_PIPEB_SELECT;
4981                 } else {
4982                         temp &= ~LVDS_PIPEB_SELECT;
4983                 }
4984                 /* set the corresponsding LVDS_BORDER bit */
4985                 temp |= dev_priv->lvds_border_bits;
4986                 /* Set the B0-B3 data pairs corresponding to whether we're going to
4987                  * set the DPLLs for dual-channel mode or not.
4988                  */
4989                 if (clock.p2 == 7)
4990                         temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
4991                 else
4992                         temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
4993
4994                 /* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
4995                  * appropriately here, but we need to look more thoroughly into how
4996                  * panels behave in the two modes.
4997                  */
4998                 /* set the dithering flag on LVDS as needed */
4999                 if (INTEL_INFO(dev)->gen >= 4) {
5000                         if (dev_priv->lvds_dither)
5001                                 temp |= LVDS_ENABLE_DITHER;
5002                         else
5003                                 temp &= ~LVDS_ENABLE_DITHER;
5004                 }
5005                 if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
5006                         lvds_sync |= LVDS_HSYNC_POLARITY;
5007                 if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
5008                         lvds_sync |= LVDS_VSYNC_POLARITY;
5009                 if ((temp & (LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY))
5010                     != lvds_sync) {
5011                         char flags[2] = "-+";
5012                         DRM_INFO("Changing LVDS panel from "
5013                                  "(%chsync, %cvsync) to (%chsync, %cvsync)\n",
5014                                  flags[!(temp & LVDS_HSYNC_POLARITY)],
5015                                  flags[!(temp & LVDS_VSYNC_POLARITY)],
5016                                  flags[!(lvds_sync & LVDS_HSYNC_POLARITY)],
5017                                  flags[!(lvds_sync & LVDS_VSYNC_POLARITY)]);
5018                         temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
5019                         temp |= lvds_sync;
5020                 }
5021                 I915_WRITE(LVDS, temp);
5022         }
5023
5024         if (is_dp) {
5025                 intel_dp_set_m_n(crtc, mode, adjusted_mode);
5026         }
5027
5028         I915_WRITE(DPLL(pipe), dpll);
5029
5030         /* Wait for the clocks to stabilize. */
5031         POSTING_READ(DPLL(pipe));
5032         udelay(150);
5033
5034         if (INTEL_INFO(dev)->gen >= 4) {
5035                 temp = 0;
5036                 if (is_sdvo) {
5037                         temp = intel_mode_get_pixel_multiplier(adjusted_mode);
5038                         if (temp > 1)
5039                                 temp = (temp - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
5040                         else
5041                                 temp = 0;
5042                 }
5043                 I915_WRITE(DPLL_MD(pipe), temp);
5044         } else {
5045                 /* The pixel multiplier can only be updated once the
5046                  * DPLL is enabled and the clocks are stable.
5047                  *
5048                  * So write it again.
5049                  */
5050                 I915_WRITE(DPLL(pipe), dpll);
5051         }
5052
5053         intel_crtc->lowfreq_avail = false;
5054         if (is_lvds && has_reduced_clock && i915_powersave) {
5055                 I915_WRITE(FP1(pipe), fp2);
5056                 intel_crtc->lowfreq_avail = true;
5057                 if (HAS_PIPE_CXSR(dev)) {
5058                         DRM_DEBUG_KMS("enabling CxSR downclocking\n");
5059                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
5060                 }
5061         } else {
5062                 I915_WRITE(FP1(pipe), fp);
5063                 if (HAS_PIPE_CXSR(dev)) {
5064                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
5065                         pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
5066                 }
5067         }
5068
5069         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5070                 pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
5071                 /* the chip adds 2 halflines automatically */
5072                 adjusted_mode->crtc_vdisplay -= 1;
5073                 adjusted_mode->crtc_vtotal -= 1;
5074                 adjusted_mode->crtc_vblank_start -= 1;
5075                 adjusted_mode->crtc_vblank_end -= 1;
5076                 adjusted_mode->crtc_vsync_end -= 1;
5077                 adjusted_mode->crtc_vsync_start -= 1;
5078         } else
5079                 pipeconf &= ~PIPECONF_INTERLACE_W_FIELD_INDICATION; /* progressive */
5080
5081         I915_WRITE(HTOTAL(pipe),
5082                    (adjusted_mode->crtc_hdisplay - 1) |
5083                    ((adjusted_mode->crtc_htotal - 1) << 16));
5084         I915_WRITE(HBLANK(pipe),
5085                    (adjusted_mode->crtc_hblank_start - 1) |
5086                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
5087         I915_WRITE(HSYNC(pipe),
5088                    (adjusted_mode->crtc_hsync_start - 1) |
5089                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
5090
5091         I915_WRITE(VTOTAL(pipe),
5092                    (adjusted_mode->crtc_vdisplay - 1) |
5093                    ((adjusted_mode->crtc_vtotal - 1) << 16));
5094         I915_WRITE(VBLANK(pipe),
5095                    (adjusted_mode->crtc_vblank_start - 1) |
5096                    ((adjusted_mode->crtc_vblank_end - 1) << 16));
5097         I915_WRITE(VSYNC(pipe),
5098                    (adjusted_mode->crtc_vsync_start - 1) |
5099                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
5100
5101         /* pipesrc and dspsize control the size that is scaled from,
5102          * which should always be the user's requested size.
5103          */
5104         I915_WRITE(DSPSIZE(plane),
5105                    ((mode->vdisplay - 1) << 16) |
5106                    (mode->hdisplay - 1));
5107         I915_WRITE(DSPPOS(plane), 0);
5108         I915_WRITE(PIPESRC(pipe),
5109                    ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
5110
5111         I915_WRITE(PIPECONF(pipe), pipeconf);
5112         POSTING_READ(PIPECONF(pipe));
5113         intel_enable_pipe(dev_priv, pipe, false);
5114
5115         intel_wait_for_vblank(dev, pipe);
5116
5117         I915_WRITE(DSPCNTR(plane), dspcntr);
5118         POSTING_READ(DSPCNTR(plane));
5119         intel_enable_plane(dev_priv, plane, pipe);
5120
5121         ret = intel_pipe_set_base(crtc, x, y, old_fb);
5122
5123         intel_update_watermarks(dev);
5124
5125         return ret;
5126 }
5127
5128 /*
5129  * Initialize reference clocks when the driver loads
5130  */
5131 void ironlake_init_pch_refclk(struct drm_device *dev)
5132 {
5133         struct drm_i915_private *dev_priv = dev->dev_private;
5134         struct drm_mode_config *mode_config = &dev->mode_config;
5135         struct intel_encoder *encoder;
5136         u32 temp;
5137         bool has_lvds = false;
5138         bool has_cpu_edp = false;
5139         bool has_pch_edp = false;
5140         bool has_panel = false;
5141         bool has_ck505 = false;
5142         bool can_ssc = false;
5143
5144         /* We need to take the global config into account */
5145         list_for_each_entry(encoder, &mode_config->encoder_list,
5146                             base.head) {
5147                 switch (encoder->type) {
5148                 case INTEL_OUTPUT_LVDS:
5149                         has_panel = true;
5150                         has_lvds = true;
5151                         break;
5152                 case INTEL_OUTPUT_EDP:
5153                         has_panel = true;
5154                         if (intel_encoder_is_pch_edp(&encoder->base))
5155                                 has_pch_edp = true;
5156                         else
5157                                 has_cpu_edp = true;
5158                         break;
5159                 }
5160         }
5161
5162         if (HAS_PCH_IBX(dev)) {
5163                 has_ck505 = dev_priv->display_clock_mode;
5164                 can_ssc = has_ck505;
5165         } else {
5166                 has_ck505 = false;
5167                 can_ssc = true;
5168         }
5169
5170         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_pch_edp %d has_cpu_edp %d has_ck505 %d\n",
5171                       has_panel, has_lvds, has_pch_edp, has_cpu_edp,
5172                       has_ck505);
5173
5174         /* Ironlake: try to setup display ref clock before DPLL
5175          * enabling. This is only under driver's control after
5176          * PCH B stepping, previous chipset stepping should be
5177          * ignoring this setting.
5178          */
5179         temp = I915_READ(PCH_DREF_CONTROL);
5180         /* Always enable nonspread source */
5181         temp &= ~DREF_NONSPREAD_SOURCE_MASK;
5182
5183         if (has_ck505)
5184                 temp |= DREF_NONSPREAD_CK505_ENABLE;
5185         else
5186                 temp |= DREF_NONSPREAD_SOURCE_ENABLE;
5187
5188         if (has_panel) {
5189                 temp &= ~DREF_SSC_SOURCE_MASK;
5190                 temp |= DREF_SSC_SOURCE_ENABLE;
5191
5192                 /* SSC must be turned on before enabling the CPU output  */
5193                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
5194                         DRM_DEBUG_KMS("Using SSC on panel\n");
5195                         temp |= DREF_SSC1_ENABLE;
5196                 }
5197
5198                 /* Get SSC going before enabling the outputs */
5199                 I915_WRITE(PCH_DREF_CONTROL, temp);
5200                 POSTING_READ(PCH_DREF_CONTROL);
5201                 udelay(200);
5202
5203                 temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
5204
5205                 /* Enable CPU source on CPU attached eDP */
5206                 if (has_cpu_edp) {
5207                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
5208                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
5209                                 temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
5210                         }
5211                         else
5212                                 temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
5213                 } else
5214                         temp |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
5215
5216                 I915_WRITE(PCH_DREF_CONTROL, temp);
5217                 POSTING_READ(PCH_DREF_CONTROL);
5218                 udelay(200);
5219         } else {
5220                 DRM_DEBUG_KMS("Disabling SSC entirely\n");
5221
5222                 temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
5223
5224                 /* Turn off CPU output */
5225                 temp |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
5226
5227                 I915_WRITE(PCH_DREF_CONTROL, temp);
5228                 POSTING_READ(PCH_DREF_CONTROL);
5229                 udelay(200);
5230
5231                 /* Turn off the SSC source */
5232                 temp &= ~DREF_SSC_SOURCE_MASK;
5233                 temp |= DREF_SSC_SOURCE_DISABLE;
5234
5235                 /* Turn off SSC1 */
5236                 temp &= ~ DREF_SSC1_ENABLE;
5237
5238                 I915_WRITE(PCH_DREF_CONTROL, temp);
5239                 POSTING_READ(PCH_DREF_CONTROL);
5240                 udelay(200);
5241         }
5242 }
5243
5244 static int ironlake_get_refclk(struct drm_crtc *crtc)
5245 {
5246         struct drm_device *dev = crtc->dev;
5247         struct drm_i915_private *dev_priv = dev->dev_private;
5248         struct intel_encoder *encoder;
5249         struct drm_mode_config *mode_config = &dev->mode_config;
5250         struct intel_encoder *edp_encoder = NULL;
5251         int num_connectors = 0;
5252         bool is_lvds = false;
5253
5254         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
5255                 if (encoder->base.crtc != crtc)
5256                         continue;
5257
5258                 switch (encoder->type) {
5259                 case INTEL_OUTPUT_LVDS:
5260                         is_lvds = true;
5261                         break;
5262                 case INTEL_OUTPUT_EDP:
5263                         edp_encoder = encoder;
5264                         break;
5265                 }
5266                 num_connectors++;
5267         }
5268
5269         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
5270                 DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
5271                               dev_priv->lvds_ssc_freq);
5272                 return dev_priv->lvds_ssc_freq * 1000;
5273         }
5274
5275         return 120000;
5276 }
5277
5278 static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
5279                                   struct drm_display_mode *mode,
5280                                   struct drm_display_mode *adjusted_mode,
5281                                   int x, int y,
5282                                   struct drm_framebuffer *old_fb)
5283 {
5284         struct drm_device *dev = crtc->dev;
5285         struct drm_i915_private *dev_priv = dev->dev_private;
5286         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5287         int pipe = intel_crtc->pipe;
5288         int plane = intel_crtc->plane;
5289         int refclk, num_connectors = 0;
5290         intel_clock_t clock, reduced_clock;
5291         u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf;
5292         bool ok, has_reduced_clock = false, is_sdvo = false;
5293         bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
5294         struct intel_encoder *has_edp_encoder = NULL;
5295         struct drm_mode_config *mode_config = &dev->mode_config;
5296         struct intel_encoder *encoder;
5297         const intel_limit_t *limit;
5298         int ret;
5299         struct fdi_m_n m_n = {0};
5300         u32 temp;
5301         u32 lvds_sync = 0;
5302         int target_clock, pixel_multiplier, lane, link_bw, factor;
5303         unsigned int pipe_bpp;
5304         bool dither;
5305
5306         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
5307                 if (encoder->base.crtc != crtc)
5308                         continue;
5309
5310                 switch (encoder->type) {
5311                 case INTEL_OUTPUT_LVDS:
5312                         is_lvds = true;
5313                         break;
5314                 case INTEL_OUTPUT_SDVO:
5315                 case INTEL_OUTPUT_HDMI:
5316                         is_sdvo = true;
5317                         if (encoder->needs_tv_clock)
5318                                 is_tv = true;
5319                         break;
5320                 case INTEL_OUTPUT_TVOUT:
5321                         is_tv = true;
5322                         break;
5323                 case INTEL_OUTPUT_ANALOG:
5324                         is_crt = true;
5325                         break;
5326                 case INTEL_OUTPUT_DISPLAYPORT:
5327                         is_dp = true;
5328                         break;
5329                 case INTEL_OUTPUT_EDP:
5330                         has_edp_encoder = encoder;
5331                         break;
5332                 }
5333
5334                 num_connectors++;
5335         }
5336
5337         refclk = ironlake_get_refclk(crtc);
5338
5339         /*
5340          * Returns a set of divisors for the desired target clock with the given
5341          * refclk, or FALSE.  The returned values represent the clock equation:
5342          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
5343          */
5344         limit = intel_limit(crtc, refclk);
5345         ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, &clock);
5346         if (!ok) {
5347                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
5348                 return -EINVAL;
5349         }
5350
5351         /* Ensure that the cursor is valid for the new mode before changing... */
5352         intel_crtc_update_cursor(crtc, true);
5353
5354         if (is_lvds && dev_priv->lvds_downclock_avail) {
5355                 has_reduced_clock = limit->find_pll(limit, crtc,
5356                                                     dev_priv->lvds_downclock,
5357                                                     refclk,
5358                                                     &reduced_clock);
5359                 if (has_reduced_clock && (clock.p != reduced_clock.p)) {
5360                         /*
5361                          * If the different P is found, it means that we can't
5362                          * switch the display clock by using the FP0/FP1.
5363                          * In such case we will disable the LVDS downclock
5364                          * feature.
5365                          */
5366                         DRM_DEBUG_KMS("Different P is found for "
5367                                       "LVDS clock/downclock\n");
5368                         has_reduced_clock = 0;
5369                 }
5370         }
5371         /* SDVO TV has fixed PLL values depend on its clock range,
5372            this mirrors vbios setting. */
5373         if (is_sdvo && is_tv) {
5374                 if (adjusted_mode->clock >= 100000
5375                     && adjusted_mode->clock < 140500) {
5376                         clock.p1 = 2;
5377                         clock.p2 = 10;
5378                         clock.n = 3;
5379                         clock.m1 = 16;
5380                         clock.m2 = 8;
5381                 } else if (adjusted_mode->clock >= 140500
5382                            && adjusted_mode->clock <= 200000) {
5383                         clock.p1 = 1;
5384                         clock.p2 = 10;
5385                         clock.n = 6;
5386                         clock.m1 = 12;
5387                         clock.m2 = 8;
5388                 }
5389         }
5390
5391         /* FDI link */
5392         pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
5393         lane = 0;
5394         /* CPU eDP doesn't require FDI link, so just set DP M/N
5395            according to current link config */
5396         if (has_edp_encoder &&
5397             !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
5398                 target_clock = mode->clock;
5399                 intel_edp_link_config(has_edp_encoder,
5400                                       &lane, &link_bw);
5401         } else {
5402                 /* [e]DP over FDI requires target mode clock
5403                    instead of link clock */
5404                 if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base))
5405                         target_clock = mode->clock;
5406                 else
5407                         target_clock = adjusted_mode->clock;
5408
5409                 /* FDI is a binary signal running at ~2.7GHz, encoding
5410                  * each output octet as 10 bits. The actual frequency
5411                  * is stored as a divider into a 100MHz clock, and the
5412                  * mode pixel clock is stored in units of 1KHz.
5413                  * Hence the bw of each lane in terms of the mode signal
5414                  * is:
5415                  */
5416                 link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
5417         }
5418
5419         /* determine panel color depth */
5420         temp = I915_READ(PIPECONF(pipe));
5421         temp &= ~PIPE_BPC_MASK;
5422         dither = intel_choose_pipe_bpp_dither(crtc, &pipe_bpp);
5423         switch (pipe_bpp) {
5424         case 18:
5425                 temp |= PIPE_6BPC;
5426                 break;
5427         case 24:
5428                 temp |= PIPE_8BPC;
5429                 break;
5430         case 30:
5431                 temp |= PIPE_10BPC;
5432                 break;
5433         case 36:
5434                 temp |= PIPE_12BPC;
5435                 break;
5436         default:
5437                 WARN(1, "intel_choose_pipe_bpp returned invalid value %d\n",
5438                         pipe_bpp);
5439                 temp |= PIPE_8BPC;
5440                 pipe_bpp = 24;
5441                 break;
5442         }
5443
5444         intel_crtc->bpp = pipe_bpp;
5445         I915_WRITE(PIPECONF(pipe), temp);
5446
5447         if (!lane) {
5448                 /*
5449                  * Account for spread spectrum to avoid
5450                  * oversubscribing the link. Max center spread
5451                  * is 2.5%; use 5% for safety's sake.
5452                  */
5453                 u32 bps = target_clock * intel_crtc->bpp * 21 / 20;
5454                 lane = bps / (link_bw * 8) + 1;
5455         }
5456
5457         intel_crtc->fdi_lanes = lane;
5458
5459         if (pixel_multiplier > 1)
5460                 link_bw *= pixel_multiplier;
5461         ironlake_compute_m_n(intel_crtc->bpp, lane, target_clock, link_bw,
5462                              &m_n);
5463
5464         fp = clock.n << 16 | clock.m1 << 8 | clock.m2;
5465         if (has_reduced_clock)
5466                 fp2 = reduced_clock.n << 16 | reduced_clock.m1 << 8 |
5467                         reduced_clock.m2;
5468
5469         /* Enable autotuning of the PLL clock (if permissible) */
5470         factor = 21;
5471         if (is_lvds) {
5472                 if ((intel_panel_use_ssc(dev_priv) &&
5473                      dev_priv->lvds_ssc_freq == 100) ||
5474                     (I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP)
5475                         factor = 25;
5476         } else if (is_sdvo && is_tv)
5477                 factor = 20;
5478
5479         if (clock.m < factor * clock.n)
5480                 fp |= FP_CB_TUNE;
5481
5482         dpll = 0;
5483
5484         if (is_lvds)
5485                 dpll |= DPLLB_MODE_LVDS;
5486         else
5487                 dpll |= DPLLB_MODE_DAC_SERIAL;
5488         if (is_sdvo) {
5489                 int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
5490                 if (pixel_multiplier > 1) {
5491                         dpll |= (pixel_multiplier - 1) << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
5492                 }
5493                 dpll |= DPLL_DVO_HIGH_SPEED;
5494         }
5495         if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base))
5496                 dpll |= DPLL_DVO_HIGH_SPEED;
5497
5498         /* compute bitmask from p1 value */
5499         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
5500         /* also FPA1 */
5501         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
5502
5503         switch (clock.p2) {
5504         case 5:
5505                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
5506                 break;
5507         case 7:
5508                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
5509                 break;
5510         case 10:
5511                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
5512                 break;
5513         case 14:
5514                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
5515                 break;
5516         }
5517
5518         if (is_sdvo && is_tv)
5519                 dpll |= PLL_REF_INPUT_TVCLKINBC;
5520         else if (is_tv)
5521                 /* XXX: just matching BIOS for now */
5522                 /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
5523                 dpll |= 3;
5524         else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
5525                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
5526         else
5527                 dpll |= PLL_REF_INPUT_DREFCLK;
5528
5529         /* setup pipeconf */
5530         pipeconf = I915_READ(PIPECONF(pipe));
5531
5532         /* Set up the display plane register */
5533         dspcntr = DISPPLANE_GAMMA_ENABLE;
5534
5535         DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B');
5536         drm_mode_debug_printmodeline(mode);
5537
5538         /* PCH eDP needs FDI, but CPU eDP does not */
5539         if (!has_edp_encoder || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
5540                 I915_WRITE(PCH_FP0(pipe), fp);
5541                 I915_WRITE(PCH_DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
5542
5543                 POSTING_READ(PCH_DPLL(pipe));
5544                 udelay(150);
5545         }
5546
5547         /* enable transcoder DPLL */
5548         if (HAS_PCH_CPT(dev)) {
5549                 temp = I915_READ(PCH_DPLL_SEL);
5550                 switch (pipe) {
5551                 case 0:
5552                         temp |= TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL;
5553                         break;
5554                 case 1:
5555                         temp |= TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL;
5556                         break;
5557                 case 2:
5558                         /* FIXME: manage transcoder PLLs? */
5559                         temp |= TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL;
5560                         break;
5561                 default:
5562                         BUG();
5563                 }
5564                 I915_WRITE(PCH_DPLL_SEL, temp);
5565
5566                 POSTING_READ(PCH_DPLL_SEL);
5567                 udelay(150);
5568         }
5569
5570         /* The LVDS pin pair needs to be on before the DPLLs are enabled.
5571          * This is an exception to the general rule that mode_set doesn't turn
5572          * things on.
5573          */
5574         if (is_lvds) {
5575                 temp = I915_READ(PCH_LVDS);
5576                 temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
5577                 if (pipe == 1) {
5578                         if (HAS_PCH_CPT(dev))
5579                                 temp |= PORT_TRANS_B_SEL_CPT;
5580                         else
5581                                 temp |= LVDS_PIPEB_SELECT;
5582                 } else {
5583                         if (HAS_PCH_CPT(dev))
5584                                 temp &= ~PORT_TRANS_SEL_MASK;
5585                         else
5586                                 temp &= ~LVDS_PIPEB_SELECT;
5587                 }
5588                 /* set the corresponsding LVDS_BORDER bit */
5589                 temp |= dev_priv->lvds_border_bits;
5590                 /* Set the B0-B3 data pairs corresponding to whether we're going to
5591                  * set the DPLLs for dual-channel mode or not.
5592                  */
5593                 if (clock.p2 == 7)
5594                         temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
5595                 else
5596                         temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
5597
5598                 /* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
5599                  * appropriately here, but we need to look more thoroughly into how
5600                  * panels behave in the two modes.
5601                  */
5602                 if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
5603                         lvds_sync |= LVDS_HSYNC_POLARITY;
5604                 if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
5605                         lvds_sync |= LVDS_VSYNC_POLARITY;
5606                 if ((temp & (LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY))
5607                     != lvds_sync) {
5608                         char flags[2] = "-+";
5609                         DRM_INFO("Changing LVDS panel from "
5610                                  "(%chsync, %cvsync) to (%chsync, %cvsync)\n",
5611                                  flags[!(temp & LVDS_HSYNC_POLARITY)],
5612                                  flags[!(temp & LVDS_VSYNC_POLARITY)],
5613                                  flags[!(lvds_sync & LVDS_HSYNC_POLARITY)],
5614                                  flags[!(lvds_sync & LVDS_VSYNC_POLARITY)]);
5615                         temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
5616                         temp |= lvds_sync;
5617                 }
5618                 I915_WRITE(PCH_LVDS, temp);
5619         }
5620
5621         pipeconf &= ~PIPECONF_DITHER_EN;
5622         pipeconf &= ~PIPECONF_DITHER_TYPE_MASK;
5623         if ((is_lvds && dev_priv->lvds_dither) || dither) {
5624                 pipeconf |= PIPECONF_DITHER_EN;
5625                 pipeconf |= PIPECONF_DITHER_TYPE_ST1;
5626         }
5627         if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
5628                 intel_dp_set_m_n(crtc, mode, adjusted_mode);
5629         } else {
5630                 /* For non-DP output, clear any trans DP clock recovery setting.*/
5631                 I915_WRITE(TRANSDATA_M1(pipe), 0);
5632                 I915_WRITE(TRANSDATA_N1(pipe), 0);
5633                 I915_WRITE(TRANSDPLINK_M1(pipe), 0);
5634                 I915_WRITE(TRANSDPLINK_N1(pipe), 0);
5635         }
5636
5637         if (!has_edp_encoder ||
5638             intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
5639                 I915_WRITE(PCH_DPLL(pipe), dpll);
5640
5641                 /* Wait for the clocks to stabilize. */
5642                 POSTING_READ(PCH_DPLL(pipe));
5643                 udelay(150);
5644
5645                 /* The pixel multiplier can only be updated once the
5646                  * DPLL is enabled and the clocks are stable.
5647                  *
5648                  * So write it again.
5649                  */
5650                 I915_WRITE(PCH_DPLL(pipe), dpll);
5651         }
5652
5653         intel_crtc->lowfreq_avail = false;
5654         if (is_lvds && has_reduced_clock && i915_powersave) {
5655                 I915_WRITE(PCH_FP1(pipe), fp2);
5656                 intel_crtc->lowfreq_avail = true;
5657                 if (HAS_PIPE_CXSR(dev)) {
5658                         DRM_DEBUG_KMS("enabling CxSR downclocking\n");
5659                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
5660                 }
5661         } else {
5662                 I915_WRITE(PCH_FP1(pipe), fp);
5663                 if (HAS_PIPE_CXSR(dev)) {
5664                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
5665                         pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
5666                 }
5667         }
5668
5669         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5670                 pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
5671                 /* the chip adds 2 halflines automatically */
5672                 adjusted_mode->crtc_vdisplay -= 1;
5673                 adjusted_mode->crtc_vtotal -= 1;
5674                 adjusted_mode->crtc_vblank_start -= 1;
5675                 adjusted_mode->crtc_vblank_end -= 1;
5676                 adjusted_mode->crtc_vsync_end -= 1;
5677                 adjusted_mode->crtc_vsync_start -= 1;
5678         } else
5679                 pipeconf &= ~PIPECONF_INTERLACE_W_FIELD_INDICATION; /* progressive */
5680
5681         I915_WRITE(HTOTAL(pipe),
5682                    (adjusted_mode->crtc_hdisplay - 1) |
5683                    ((adjusted_mode->crtc_htotal - 1) << 16));
5684         I915_WRITE(HBLANK(pipe),
5685                    (adjusted_mode->crtc_hblank_start - 1) |
5686                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
5687         I915_WRITE(HSYNC(pipe),
5688                    (adjusted_mode->crtc_hsync_start - 1) |
5689                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
5690
5691         I915_WRITE(VTOTAL(pipe),
5692                    (adjusted_mode->crtc_vdisplay - 1) |
5693                    ((adjusted_mode->crtc_vtotal - 1) << 16));
5694         I915_WRITE(VBLANK(pipe),
5695                    (adjusted_mode->crtc_vblank_start - 1) |
5696                    ((adjusted_mode->crtc_vblank_end - 1) << 16));
5697         I915_WRITE(VSYNC(pipe),
5698                    (adjusted_mode->crtc_vsync_start - 1) |
5699                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
5700
5701         /* pipesrc controls the size that is scaled from, which should
5702          * always be the user's requested size.
5703          */
5704         I915_WRITE(PIPESRC(pipe),
5705                    ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
5706
5707         I915_WRITE(PIPE_DATA_M1(pipe), TU_SIZE(m_n.tu) | m_n.gmch_m);
5708         I915_WRITE(PIPE_DATA_N1(pipe), m_n.gmch_n);
5709         I915_WRITE(PIPE_LINK_M1(pipe), m_n.link_m);
5710         I915_WRITE(PIPE_LINK_N1(pipe), m_n.link_n);
5711
5712         if (has_edp_encoder &&
5713             !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
5714                 ironlake_set_pll_edp(crtc, adjusted_mode->clock);
5715         }
5716
5717         I915_WRITE(PIPECONF(pipe), pipeconf);
5718         POSTING_READ(PIPECONF(pipe));
5719
5720         intel_wait_for_vblank(dev, pipe);
5721
5722         if (IS_GEN5(dev)) {
5723                 /* enable address swizzle for tiling buffer */
5724                 temp = I915_READ(DISP_ARB_CTL);
5725                 I915_WRITE(DISP_ARB_CTL, temp | DISP_TILE_SURFACE_SWIZZLING);
5726         }
5727
5728         I915_WRITE(DSPCNTR(plane), dspcntr);
5729         POSTING_READ(DSPCNTR(plane));
5730
5731         ret = intel_pipe_set_base(crtc, x, y, old_fb);
5732
5733         intel_update_watermarks(dev);
5734
5735         return ret;
5736 }
5737
5738 static int intel_crtc_mode_set(struct drm_crtc *crtc,
5739                                struct drm_display_mode *mode,
5740                                struct drm_display_mode *adjusted_mode,
5741                                int x, int y,
5742                                struct drm_framebuffer *old_fb)
5743 {
5744         struct drm_device *dev = crtc->dev;
5745         struct drm_i915_private *dev_priv = dev->dev_private;
5746         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5747         int pipe = intel_crtc->pipe;
5748         int ret;
5749
5750         drm_vblank_pre_modeset(dev, pipe);
5751
5752         ret = dev_priv->display.crtc_mode_set(crtc, mode, adjusted_mode,
5753                                               x, y, old_fb);
5754
5755         drm_vblank_post_modeset(dev, pipe);
5756
5757         intel_crtc->dpms_mode = DRM_MODE_DPMS_ON;
5758
5759         return ret;
5760 }
5761
5762 static void g4x_write_eld(struct drm_connector *connector,
5763                           struct drm_crtc *crtc)
5764 {
5765         struct drm_i915_private *dev_priv = connector->dev->dev_private;
5766         uint8_t *eld = connector->eld;
5767         uint32_t eldv;
5768         uint32_t len;
5769         uint32_t i;
5770
5771         i = I915_READ(G4X_AUD_VID_DID);
5772
5773         if (i == INTEL_AUDIO_DEVBLC || i == INTEL_AUDIO_DEVCL)
5774                 eldv = G4X_ELDV_DEVCL_DEVBLC;
5775         else
5776                 eldv = G4X_ELDV_DEVCTG;
5777
5778         i = I915_READ(G4X_AUD_CNTL_ST);
5779         i &= ~(eldv | G4X_ELD_ADDR);
5780         len = (i >> 9) & 0x1f;          /* ELD buffer size */
5781         I915_WRITE(G4X_AUD_CNTL_ST, i);
5782
5783         if (!eld[0])
5784                 return;
5785
5786         len = min_t(uint8_t, eld[2], len);
5787         DRM_DEBUG_DRIVER("ELD size %d\n", len);
5788         for (i = 0; i < len; i++)
5789                 I915_WRITE(G4X_HDMIW_HDMIEDID, *((uint32_t *)eld + i));
5790
5791         i = I915_READ(G4X_AUD_CNTL_ST);
5792         i |= eldv;
5793         I915_WRITE(G4X_AUD_CNTL_ST, i);
5794 }
5795
5796 static void ironlake_write_eld(struct drm_connector *connector,
5797                                      struct drm_crtc *crtc)
5798 {
5799         struct drm_i915_private *dev_priv = connector->dev->dev_private;
5800         uint8_t *eld = connector->eld;
5801         uint32_t eldv;
5802         uint32_t i;
5803         int len;
5804         int hdmiw_hdmiedid;
5805         int aud_cntl_st;
5806         int aud_cntrl_st2;
5807
5808         if (IS_IVYBRIDGE(connector->dev)) {
5809                 hdmiw_hdmiedid = GEN7_HDMIW_HDMIEDID_A;
5810                 aud_cntl_st = GEN7_AUD_CNTRL_ST_A;
5811                 aud_cntrl_st2 = GEN7_AUD_CNTRL_ST2;
5812         } else {
5813                 hdmiw_hdmiedid = GEN5_HDMIW_HDMIEDID_A;
5814                 aud_cntl_st = GEN5_AUD_CNTL_ST_A;
5815                 aud_cntrl_st2 = GEN5_AUD_CNTL_ST2;
5816         }
5817
5818         i = to_intel_crtc(crtc)->pipe;
5819         hdmiw_hdmiedid += i * 0x100;
5820         aud_cntl_st += i * 0x100;
5821
5822         DRM_DEBUG_DRIVER("ELD on pipe %c\n", pipe_name(i));
5823
5824         i = I915_READ(aud_cntl_st);
5825         i = (i >> 29) & 0x3;            /* DIP_Port_Select, 0x1 = PortB */
5826         if (!i) {
5827                 DRM_DEBUG_DRIVER("Audio directed to unknown port\n");
5828                 /* operate blindly on all ports */
5829                 eldv = GEN5_ELD_VALIDB;
5830                 eldv |= GEN5_ELD_VALIDB << 4;
5831                 eldv |= GEN5_ELD_VALIDB << 8;
5832         } else {
5833                 DRM_DEBUG_DRIVER("ELD on port %c\n", 'A' + i);
5834                 eldv = GEN5_ELD_VALIDB << ((i - 1) * 4);
5835         }
5836
5837         i = I915_READ(aud_cntrl_st2);
5838         i &= ~eldv;
5839         I915_WRITE(aud_cntrl_st2, i);
5840
5841         if (!eld[0])
5842                 return;
5843
5844         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
5845                 DRM_DEBUG_DRIVER("ELD: DisplayPort detected\n");
5846                 eld[5] |= (1 << 2);     /* Conn_Type, 0x1 = DisplayPort */
5847         }
5848
5849         i = I915_READ(aud_cntl_st);
5850         i &= ~GEN5_ELD_ADDRESS;
5851         I915_WRITE(aud_cntl_st, i);
5852
5853         len = min_t(uint8_t, eld[2], 21);       /* 84 bytes of hw ELD buffer */
5854         DRM_DEBUG_DRIVER("ELD size %d\n", len);
5855         for (i = 0; i < len; i++)
5856                 I915_WRITE(hdmiw_hdmiedid, *((uint32_t *)eld + i));
5857
5858         i = I915_READ(aud_cntrl_st2);
5859         i |= eldv;
5860         I915_WRITE(aud_cntrl_st2, i);
5861 }
5862
5863 void intel_write_eld(struct drm_encoder *encoder,
5864                      struct drm_display_mode *mode)
5865 {
5866         struct drm_crtc *crtc = encoder->crtc;
5867         struct drm_connector *connector;
5868         struct drm_device *dev = encoder->dev;
5869         struct drm_i915_private *dev_priv = dev->dev_private;
5870
5871         connector = drm_select_eld(encoder, mode);
5872         if (!connector)
5873                 return;
5874
5875         DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
5876                          connector->base.id,
5877                          drm_get_connector_name(connector),
5878                          connector->encoder->base.id,
5879                          drm_get_encoder_name(connector->encoder));
5880
5881         connector->eld[6] = drm_av_sync_delay(connector, mode) / 2;
5882
5883         if (dev_priv->display.write_eld)
5884                 dev_priv->display.write_eld(connector, crtc);
5885 }
5886
5887 /** Loads the palette/gamma unit for the CRTC with the prepared values */
5888 void intel_crtc_load_lut(struct drm_crtc *crtc)
5889 {
5890         struct drm_device *dev = crtc->dev;
5891         struct drm_i915_private *dev_priv = dev->dev_private;
5892         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5893         int palreg = PALETTE(intel_crtc->pipe);
5894         int i;
5895
5896         /* The clocks have to be on to load the palette. */
5897         if (!crtc->enabled)
5898                 return;
5899
5900         /* use legacy palette for Ironlake */
5901         if (HAS_PCH_SPLIT(dev))
5902                 palreg = LGC_PALETTE(intel_crtc->pipe);
5903
5904         for (i = 0; i < 256; i++) {
5905                 I915_WRITE(palreg + 4 * i,
5906                            (intel_crtc->lut_r[i] << 16) |
5907                            (intel_crtc->lut_g[i] << 8) |
5908                            intel_crtc->lut_b[i]);
5909         }
5910 }
5911
5912 static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
5913 {
5914         struct drm_device *dev = crtc->dev;
5915         struct drm_i915_private *dev_priv = dev->dev_private;
5916         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5917         bool visible = base != 0;
5918         u32 cntl;
5919
5920         if (intel_crtc->cursor_visible == visible)
5921                 return;
5922
5923         cntl = I915_READ(_CURACNTR);
5924         if (visible) {
5925                 /* On these chipsets we can only modify the base whilst
5926                  * the cursor is disabled.
5927                  */
5928                 I915_WRITE(_CURABASE, base);
5929
5930                 cntl &= ~(CURSOR_FORMAT_MASK);
5931                 /* XXX width must be 64, stride 256 => 0x00 << 28 */
5932                 cntl |= CURSOR_ENABLE |
5933                         CURSOR_GAMMA_ENABLE |
5934                         CURSOR_FORMAT_ARGB;
5935         } else
5936                 cntl &= ~(CURSOR_ENABLE | CURSOR_GAMMA_ENABLE);
5937         I915_WRITE(_CURACNTR, cntl);
5938
5939         intel_crtc->cursor_visible = visible;
5940 }
5941
5942 static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
5943 {
5944         struct drm_device *dev = crtc->dev;
5945         struct drm_i915_private *dev_priv = dev->dev_private;
5946         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5947         int pipe = intel_crtc->pipe;
5948         bool visible = base != 0;
5949
5950         if (intel_crtc->cursor_visible != visible) {
5951                 uint32_t cntl = I915_READ(CURCNTR(pipe));
5952                 if (base) {
5953                         cntl &= ~(CURSOR_MODE | MCURSOR_PIPE_SELECT);
5954                         cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
5955                         cntl |= pipe << 28; /* Connect to correct pipe */
5956                 } else {
5957                         cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
5958                         cntl |= CURSOR_MODE_DISABLE;
5959                 }
5960                 I915_WRITE(CURCNTR(pipe), cntl);
5961
5962                 intel_crtc->cursor_visible = visible;
5963         }
5964         /* and commit changes on next vblank */
5965         I915_WRITE(CURBASE(pipe), base);
5966 }
5967
5968 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
5969 static void intel_crtc_update_cursor(struct drm_crtc *crtc,
5970                                      bool on)
5971 {
5972         struct drm_device *dev = crtc->dev;
5973         struct drm_i915_private *dev_priv = dev->dev_private;
5974         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5975         int pipe = intel_crtc->pipe;
5976         int x = intel_crtc->cursor_x;
5977         int y = intel_crtc->cursor_y;
5978         u32 base, pos;
5979         bool visible;
5980
5981         pos = 0;
5982
5983         if (on && crtc->enabled && crtc->fb) {
5984                 base = intel_crtc->cursor_addr;
5985                 if (x > (int) crtc->fb->width)
5986                         base = 0;
5987
5988                 if (y > (int) crtc->fb->height)
5989                         base = 0;
5990         } else
5991                 base = 0;
5992
5993         if (x < 0) {
5994                 if (x + intel_crtc->cursor_width < 0)
5995                         base = 0;
5996
5997                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
5998                 x = -x;
5999         }
6000         pos |= x << CURSOR_X_SHIFT;
6001
6002         if (y < 0) {
6003                 if (y + intel_crtc->cursor_height < 0)
6004                         base = 0;
6005
6006                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
6007                 y = -y;
6008         }
6009         pos |= y << CURSOR_Y_SHIFT;
6010
6011         visible = base != 0;
6012         if (!visible && !intel_crtc->cursor_visible)
6013                 return;
6014
6015         I915_WRITE(CURPOS(pipe), pos);
6016         if (IS_845G(dev) || IS_I865G(dev))
6017                 i845_update_cursor(crtc, base);
6018         else
6019                 i9xx_update_cursor(crtc, base);
6020
6021         if (visible)
6022                 intel_mark_busy(dev, to_intel_framebuffer(crtc->fb)->obj);
6023 }
6024
6025 static int intel_crtc_cursor_set(struct drm_crtc *crtc,
6026                                  struct drm_file *file,
6027                                  uint32_t handle,
6028                                  uint32_t width, uint32_t height)
6029 {
6030         struct drm_device *dev = crtc->dev;
6031         struct drm_i915_private *dev_priv = dev->dev_private;
6032         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6033         struct drm_i915_gem_object *obj;
6034         uint32_t addr;
6035         int ret;
6036
6037         DRM_DEBUG_KMS("\n");
6038
6039         /* if we want to turn off the cursor ignore width and height */
6040         if (!handle) {
6041                 DRM_DEBUG_KMS("cursor off\n");
6042                 addr = 0;
6043                 obj = NULL;
6044                 mutex_lock(&dev->struct_mutex);
6045                 goto finish;
6046         }
6047
6048         /* Currently we only support 64x64 cursors */
6049         if (width != 64 || height != 64) {
6050                 DRM_ERROR("we currently only support 64x64 cursors\n");
6051                 return -EINVAL;
6052         }
6053
6054         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
6055         if (&obj->base == NULL)
6056                 return -ENOENT;
6057
6058         if (obj->base.size < width * height * 4) {
6059                 DRM_ERROR("buffer is to small\n");
6060                 ret = -ENOMEM;
6061                 goto fail;
6062         }
6063
6064         /* we only need to pin inside GTT if cursor is non-phy */
6065         mutex_lock(&dev->struct_mutex);
6066         if (!dev_priv->info->cursor_needs_physical) {
6067                 if (obj->tiling_mode) {
6068                         DRM_ERROR("cursor cannot be tiled\n");
6069                         ret = -EINVAL;
6070                         goto fail_locked;
6071                 }
6072
6073                 ret = i915_gem_object_pin_to_display_plane(obj, 0, NULL);
6074                 if (ret) {
6075                         DRM_ERROR("failed to move cursor bo into the GTT\n");
6076                         goto fail_locked;
6077                 }
6078
6079                 ret = i915_gem_object_put_fence(obj);
6080                 if (ret) {
6081                         DRM_ERROR("failed to release fence for cursor");
6082                         goto fail_unpin;
6083                 }
6084
6085                 addr = obj->gtt_offset;
6086         } else {
6087                 int align = IS_I830(dev) ? 16 * 1024 : 256;
6088                 ret = i915_gem_attach_phys_object(dev, obj,
6089                                                   (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1,
6090                                                   align);
6091                 if (ret) {
6092                         DRM_ERROR("failed to attach phys object\n");
6093                         goto fail_locked;
6094                 }
6095                 addr = obj->phys_obj->handle->busaddr;
6096         }
6097
6098         if (IS_GEN2(dev))
6099                 I915_WRITE(CURSIZE, (height << 12) | width);
6100
6101  finish:
6102         if (intel_crtc->cursor_bo) {
6103                 if (dev_priv->info->cursor_needs_physical) {
6104                         if (intel_crtc->cursor_bo != obj)
6105                                 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
6106                 } else
6107                         i915_gem_object_unpin(intel_crtc->cursor_bo);
6108                 drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
6109         }
6110
6111         mutex_unlock(&dev->struct_mutex);
6112
6113         intel_crtc->cursor_addr = addr;
6114         intel_crtc->cursor_bo = obj;
6115         intel_crtc->cursor_width = width;
6116         intel_crtc->cursor_height = height;
6117
6118         intel_crtc_update_cursor(crtc, true);
6119
6120         return 0;
6121 fail_unpin:
6122         i915_gem_object_unpin(obj);
6123 fail_locked:
6124         mutex_unlock(&dev->struct_mutex);
6125 fail:
6126         drm_gem_object_unreference_unlocked(&obj->base);
6127         return ret;
6128 }
6129
6130 static int intel_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
6131 {
6132         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6133
6134         intel_crtc->cursor_x = x;
6135         intel_crtc->cursor_y = y;
6136
6137         intel_crtc_update_cursor(crtc, true);
6138
6139         return 0;
6140 }
6141
6142 /** Sets the color ramps on behalf of RandR */
6143 void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
6144                                  u16 blue, int regno)
6145 {
6146         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6147
6148         intel_crtc->lut_r[regno] = red >> 8;
6149         intel_crtc->lut_g[regno] = green >> 8;
6150         intel_crtc->lut_b[regno] = blue >> 8;
6151 }
6152
6153 void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
6154                              u16 *blue, int regno)
6155 {
6156         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6157
6158         *red = intel_crtc->lut_r[regno] << 8;
6159         *green = intel_crtc->lut_g[regno] << 8;
6160         *blue = intel_crtc->lut_b[regno] << 8;
6161 }
6162
6163 static void intel_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
6164                                  u16 *blue, uint32_t start, uint32_t size)
6165 {
6166         int end = (start + size > 256) ? 256 : start + size, i;
6167         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6168
6169         for (i = start; i < end; i++) {
6170                 intel_crtc->lut_r[i] = red[i] >> 8;
6171                 intel_crtc->lut_g[i] = green[i] >> 8;
6172                 intel_crtc->lut_b[i] = blue[i] >> 8;
6173         }
6174
6175         intel_crtc_load_lut(crtc);
6176 }
6177
6178 /**
6179  * Get a pipe with a simple mode set on it for doing load-based monitor
6180  * detection.
6181  *
6182  * It will be up to the load-detect code to adjust the pipe as appropriate for
6183  * its requirements.  The pipe will be connected to no other encoders.
6184  *
6185  * Currently this code will only succeed if there is a pipe with no encoders
6186  * configured for it.  In the future, it could choose to temporarily disable
6187  * some outputs to free up a pipe for its use.
6188  *
6189  * \return crtc, or NULL if no pipes are available.
6190  */
6191
6192 /* VESA 640x480x72Hz mode to set on the pipe */
6193 static struct drm_display_mode load_detect_mode = {
6194         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
6195                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
6196 };
6197
6198 static struct drm_framebuffer *
6199 intel_framebuffer_create(struct drm_device *dev,
6200                          struct drm_mode_fb_cmd *mode_cmd,
6201                          struct drm_i915_gem_object *obj)
6202 {
6203         struct intel_framebuffer *intel_fb;
6204         int ret;
6205
6206         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
6207         if (!intel_fb) {
6208                 drm_gem_object_unreference_unlocked(&obj->base);
6209                 return ERR_PTR(-ENOMEM);
6210         }
6211
6212         ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
6213         if (ret) {
6214                 drm_gem_object_unreference_unlocked(&obj->base);
6215                 kfree(intel_fb);
6216                 return ERR_PTR(ret);
6217         }
6218
6219         return &intel_fb->base;
6220 }
6221
6222 static u32
6223 intel_framebuffer_pitch_for_width(int width, int bpp)
6224 {
6225         u32 pitch = DIV_ROUND_UP(width * bpp, 8);
6226         return ALIGN(pitch, 64);
6227 }
6228
6229 static u32
6230 intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp)
6231 {
6232         u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
6233         return ALIGN(pitch * mode->vdisplay, PAGE_SIZE);
6234 }
6235
6236 static struct drm_framebuffer *
6237 intel_framebuffer_create_for_mode(struct drm_device *dev,
6238                                   struct drm_display_mode *mode,
6239                                   int depth, int bpp)
6240 {
6241         struct drm_i915_gem_object *obj;
6242         struct drm_mode_fb_cmd mode_cmd;
6243
6244         obj = i915_gem_alloc_object(dev,
6245                                     intel_framebuffer_size_for_mode(mode, bpp));
6246         if (obj == NULL)
6247                 return ERR_PTR(-ENOMEM);
6248
6249         mode_cmd.width = mode->hdisplay;
6250         mode_cmd.height = mode->vdisplay;
6251         mode_cmd.depth = depth;
6252         mode_cmd.bpp = bpp;
6253         mode_cmd.pitch = intel_framebuffer_pitch_for_width(mode_cmd.width, bpp);
6254
6255         return intel_framebuffer_create(dev, &mode_cmd, obj);
6256 }
6257
6258 static struct drm_framebuffer *
6259 mode_fits_in_fbdev(struct drm_device *dev,
6260                    struct drm_display_mode *mode)
6261 {
6262         struct drm_i915_private *dev_priv = dev->dev_private;
6263         struct drm_i915_gem_object *obj;
6264         struct drm_framebuffer *fb;
6265
6266         if (dev_priv->fbdev == NULL)
6267                 return NULL;
6268
6269         obj = dev_priv->fbdev->ifb.obj;
6270         if (obj == NULL)
6271                 return NULL;
6272
6273         fb = &dev_priv->fbdev->ifb.base;
6274         if (fb->pitch < intel_framebuffer_pitch_for_width(mode->hdisplay,
6275                                                           fb->bits_per_pixel))
6276                 return NULL;
6277
6278         if (obj->base.size < mode->vdisplay * fb->pitch)
6279                 return NULL;
6280
6281         return fb;
6282 }
6283
6284 bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
6285                                 struct drm_connector *connector,
6286                                 struct drm_display_mode *mode,
6287                                 struct intel_load_detect_pipe *old)
6288 {
6289         struct intel_crtc *intel_crtc;
6290         struct drm_crtc *possible_crtc;
6291         struct drm_encoder *encoder = &intel_encoder->base;
6292         struct drm_crtc *crtc = NULL;
6293         struct drm_device *dev = encoder->dev;
6294         struct drm_framebuffer *old_fb;
6295         int i = -1;
6296
6297         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
6298                       connector->base.id, drm_get_connector_name(connector),
6299                       encoder->base.id, drm_get_encoder_name(encoder));
6300
6301         /*
6302          * Algorithm gets a little messy:
6303          *
6304          *   - if the connector already has an assigned crtc, use it (but make
6305          *     sure it's on first)
6306          *
6307          *   - try to find the first unused crtc that can drive this connector,
6308          *     and use that if we find one
6309          */
6310
6311         /* See if we already have a CRTC for this connector */
6312         if (encoder->crtc) {
6313                 crtc = encoder->crtc;
6314
6315                 intel_crtc = to_intel_crtc(crtc);
6316                 old->dpms_mode = intel_crtc->dpms_mode;
6317                 old->load_detect_temp = false;
6318
6319                 /* Make sure the crtc and connector are running */
6320                 if (intel_crtc->dpms_mode != DRM_MODE_DPMS_ON) {
6321                         struct drm_encoder_helper_funcs *encoder_funcs;
6322                         struct drm_crtc_helper_funcs *crtc_funcs;
6323
6324                         crtc_funcs = crtc->helper_private;
6325                         crtc_funcs->dpms(crtc, DRM_MODE_DPMS_ON);
6326
6327                         encoder_funcs = encoder->helper_private;
6328                         encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
6329                 }
6330
6331                 return true;
6332         }
6333
6334         /* Find an unused one (if possible) */
6335         list_for_each_entry(possible_crtc, &dev->mode_config.crtc_list, head) {
6336                 i++;
6337                 if (!(encoder->possible_crtcs & (1 << i)))
6338                         continue;
6339                 if (!possible_crtc->enabled) {
6340                         crtc = possible_crtc;
6341                         break;
6342                 }
6343         }
6344
6345         /*
6346          * If we didn't find an unused CRTC, don't use any.
6347          */
6348         if (!crtc) {
6349                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
6350                 return false;
6351         }
6352
6353         encoder->crtc = crtc;
6354         connector->encoder = encoder;
6355
6356         intel_crtc = to_intel_crtc(crtc);
6357         old->dpms_mode = intel_crtc->dpms_mode;
6358         old->load_detect_temp = true;
6359         old->release_fb = NULL;
6360
6361         if (!mode)
6362                 mode = &load_detect_mode;
6363
6364         old_fb = crtc->fb;
6365
6366         /* We need a framebuffer large enough to accommodate all accesses
6367          * that the plane may generate whilst we perform load detection.
6368          * We can not rely on the fbcon either being present (we get called
6369          * during its initialisation to detect all boot displays, or it may
6370          * not even exist) or that it is large enough to satisfy the
6371          * requested mode.
6372          */
6373         crtc->fb = mode_fits_in_fbdev(dev, mode);
6374         if (crtc->fb == NULL) {
6375                 DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
6376                 crtc->fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
6377                 old->release_fb = crtc->fb;
6378         } else
6379                 DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
6380         if (IS_ERR(crtc->fb)) {
6381                 DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
6382                 crtc->fb = old_fb;
6383                 return false;
6384         }
6385
6386         if (!drm_crtc_helper_set_mode(crtc, mode, 0, 0, old_fb)) {
6387                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
6388                 if (old->release_fb)
6389                         old->release_fb->funcs->destroy(old->release_fb);
6390                 crtc->fb = old_fb;
6391                 return false;
6392         }
6393
6394         /* let the connector get through one full cycle before testing */
6395         intel_wait_for_vblank(dev, intel_crtc->pipe);
6396
6397         return true;
6398 }
6399
6400 void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
6401                                     struct drm_connector *connector,
6402                                     struct intel_load_detect_pipe *old)
6403 {
6404         struct drm_encoder *encoder = &intel_encoder->base;
6405         struct drm_device *dev = encoder->dev;
6406         struct drm_crtc *crtc = encoder->crtc;
6407         struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
6408         struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
6409
6410         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
6411                       connector->base.id, drm_get_connector_name(connector),
6412                       encoder->base.id, drm_get_encoder_name(encoder));
6413
6414         if (old->load_detect_temp) {
6415                 connector->encoder = NULL;
6416                 drm_helper_disable_unused_functions(dev);
6417
6418                 if (old->release_fb)
6419                         old->release_fb->funcs->destroy(old->release_fb);
6420
6421                 return;
6422         }
6423
6424         /* Switch crtc and encoder back off if necessary */
6425         if (old->dpms_mode != DRM_MODE_DPMS_ON) {
6426                 encoder_funcs->dpms(encoder, old->dpms_mode);
6427                 crtc_funcs->dpms(crtc, old->dpms_mode);
6428         }
6429 }
6430
6431 /* Returns the clock of the currently programmed mode of the given pipe. */
6432 static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc)
6433 {
6434         struct drm_i915_private *dev_priv = dev->dev_private;
6435         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6436         int pipe = intel_crtc->pipe;
6437         u32 dpll = I915_READ(DPLL(pipe));
6438         u32 fp;
6439         intel_clock_t clock;
6440
6441         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
6442                 fp = I915_READ(FP0(pipe));
6443         else
6444                 fp = I915_READ(FP1(pipe));
6445
6446         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
6447         if (IS_PINEVIEW(dev)) {
6448                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
6449                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
6450         } else {
6451                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
6452                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
6453         }
6454
6455         if (!IS_GEN2(dev)) {
6456                 if (IS_PINEVIEW(dev))
6457                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
6458                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
6459                 else
6460                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
6461                                DPLL_FPA01_P1_POST_DIV_SHIFT);
6462
6463                 switch (dpll & DPLL_MODE_MASK) {
6464                 case DPLLB_MODE_DAC_SERIAL:
6465                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
6466                                 5 : 10;
6467                         break;
6468                 case DPLLB_MODE_LVDS:
6469                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
6470                                 7 : 14;
6471                         break;
6472                 default:
6473                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
6474                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
6475                         return 0;
6476                 }
6477
6478                 /* XXX: Handle the 100Mhz refclk */
6479                 intel_clock(dev, 96000, &clock);
6480         } else {
6481                 bool is_lvds = (pipe == 1) && (I915_READ(LVDS) & LVDS_PORT_EN);
6482
6483                 if (is_lvds) {
6484                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
6485                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
6486                         clock.p2 = 14;
6487
6488                         if ((dpll & PLL_REF_INPUT_MASK) ==
6489                             PLLB_REF_INPUT_SPREADSPECTRUMIN) {
6490                                 /* XXX: might not be 66MHz */
6491                                 intel_clock(dev, 66000, &clock);
6492                         } else
6493                                 intel_clock(dev, 48000, &clock);
6494                 } else {
6495                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
6496                                 clock.p1 = 2;
6497                         else {
6498                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
6499                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
6500                         }
6501                         if (dpll & PLL_P2_DIVIDE_BY_4)
6502                                 clock.p2 = 4;
6503                         else
6504                                 clock.p2 = 2;
6505
6506                         intel_clock(dev, 48000, &clock);
6507                 }
6508         }
6509
6510         /* XXX: It would be nice to validate the clocks, but we can't reuse
6511          * i830PllIsValid() because it relies on the xf86_config connector
6512          * configuration being accurate, which it isn't necessarily.
6513          */
6514
6515         return clock.dot;
6516 }
6517
6518 /** Returns the currently programmed mode of the given pipe. */
6519 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
6520                                              struct drm_crtc *crtc)
6521 {
6522         struct drm_i915_private *dev_priv = dev->dev_private;
6523         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6524         int pipe = intel_crtc->pipe;
6525         struct drm_display_mode *mode;
6526         int htot = I915_READ(HTOTAL(pipe));
6527         int hsync = I915_READ(HSYNC(pipe));
6528         int vtot = I915_READ(VTOTAL(pipe));
6529         int vsync = I915_READ(VSYNC(pipe));
6530
6531         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
6532         if (!mode)
6533                 return NULL;
6534
6535         mode->clock = intel_crtc_clock_get(dev, crtc);
6536         mode->hdisplay = (htot & 0xffff) + 1;
6537         mode->htotal = ((htot & 0xffff0000) >> 16) + 1;
6538         mode->hsync_start = (hsync & 0xffff) + 1;
6539         mode->hsync_end = ((hsync & 0xffff0000) >> 16) + 1;
6540         mode->vdisplay = (vtot & 0xffff) + 1;
6541         mode->vtotal = ((vtot & 0xffff0000) >> 16) + 1;
6542         mode->vsync_start = (vsync & 0xffff) + 1;
6543         mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1;
6544
6545         drm_mode_set_name(mode);
6546         drm_mode_set_crtcinfo(mode, 0);
6547
6548         return mode;
6549 }
6550
6551 #define GPU_IDLE_TIMEOUT 500 /* ms */
6552
6553 /* When this timer fires, we've been idle for awhile */
6554 static void intel_gpu_idle_timer(unsigned long arg)
6555 {
6556         struct drm_device *dev = (struct drm_device *)arg;
6557         drm_i915_private_t *dev_priv = dev->dev_private;
6558
6559         if (!list_empty(&dev_priv->mm.active_list)) {
6560                 /* Still processing requests, so just re-arm the timer. */
6561                 mod_timer(&dev_priv->idle_timer, jiffies +
6562                           msecs_to_jiffies(GPU_IDLE_TIMEOUT));
6563                 return;
6564         }
6565
6566         dev_priv->busy = false;
6567         queue_work(dev_priv->wq, &dev_priv->idle_work);
6568 }
6569
6570 #define CRTC_IDLE_TIMEOUT 1000 /* ms */
6571
6572 static void intel_crtc_idle_timer(unsigned long arg)
6573 {
6574         struct intel_crtc *intel_crtc = (struct intel_crtc *)arg;
6575         struct drm_crtc *crtc = &intel_crtc->base;
6576         drm_i915_private_t *dev_priv = crtc->dev->dev_private;
6577         struct intel_framebuffer *intel_fb;
6578
6579         intel_fb = to_intel_framebuffer(crtc->fb);
6580         if (intel_fb && intel_fb->obj->active) {
6581                 /* The framebuffer is still being accessed by the GPU. */
6582                 mod_timer(&intel_crtc->idle_timer, jiffies +
6583                           msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
6584                 return;
6585         }
6586
6587         intel_crtc->busy = false;
6588         queue_work(dev_priv->wq, &dev_priv->idle_work);
6589 }
6590
6591 static void intel_increase_pllclock(struct drm_crtc *crtc)
6592 {
6593         struct drm_device *dev = crtc->dev;
6594         drm_i915_private_t *dev_priv = dev->dev_private;
6595         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6596         int pipe = intel_crtc->pipe;
6597         int dpll_reg = DPLL(pipe);
6598         int dpll;
6599
6600         if (HAS_PCH_SPLIT(dev))
6601                 return;
6602
6603         if (!dev_priv->lvds_downclock_avail)
6604                 return;
6605
6606         dpll = I915_READ(dpll_reg);
6607         if (!HAS_PIPE_CXSR(dev) && (dpll & DISPLAY_RATE_SELECT_FPA1)) {
6608                 DRM_DEBUG_DRIVER("upclocking LVDS\n");
6609
6610                 /* Unlock panel regs */
6611                 I915_WRITE(PP_CONTROL,
6612                            I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS);
6613
6614                 dpll &= ~DISPLAY_RATE_SELECT_FPA1;
6615                 I915_WRITE(dpll_reg, dpll);
6616                 intel_wait_for_vblank(dev, pipe);
6617
6618                 dpll = I915_READ(dpll_reg);
6619                 if (dpll & DISPLAY_RATE_SELECT_FPA1)
6620                         DRM_DEBUG_DRIVER("failed to upclock LVDS!\n");
6621
6622                 /* ...and lock them again */
6623                 I915_WRITE(PP_CONTROL, I915_READ(PP_CONTROL) & 0x3);
6624         }
6625
6626         /* Schedule downclock */
6627         mod_timer(&intel_crtc->idle_timer, jiffies +
6628                   msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
6629 }
6630
6631 static void intel_decrease_pllclock(struct drm_crtc *crtc)
6632 {
6633         struct drm_device *dev = crtc->dev;
6634         drm_i915_private_t *dev_priv = dev->dev_private;
6635         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6636         int pipe = intel_crtc->pipe;
6637         int dpll_reg = DPLL(pipe);
6638         int dpll = I915_READ(dpll_reg);
6639
6640         if (HAS_PCH_SPLIT(dev))
6641                 return;
6642
6643         if (!dev_priv->lvds_downclock_avail)
6644                 return;
6645
6646         /*
6647          * Since this is called by a timer, we should never get here in
6648          * the manual case.
6649          */
6650         if (!HAS_PIPE_CXSR(dev) && intel_crtc->lowfreq_avail) {
6651                 DRM_DEBUG_DRIVER("downclocking LVDS\n");
6652
6653                 /* Unlock panel regs */
6654                 I915_WRITE(PP_CONTROL, I915_READ(PP_CONTROL) |
6655                            PANEL_UNLOCK_REGS);
6656
6657                 dpll |= DISPLAY_RATE_SELECT_FPA1;
6658                 I915_WRITE(dpll_reg, dpll);
6659                 intel_wait_for_vblank(dev, pipe);
6660                 dpll = I915_READ(dpll_reg);
6661                 if (!(dpll & DISPLAY_RATE_SELECT_FPA1))
6662                         DRM_DEBUG_DRIVER("failed to downclock LVDS!\n");
6663
6664                 /* ...and lock them again */
6665                 I915_WRITE(PP_CONTROL, I915_READ(PP_CONTROL) & 0x3);
6666         }
6667
6668 }
6669
6670 /**
6671  * intel_idle_update - adjust clocks for idleness
6672  * @work: work struct
6673  *
6674  * Either the GPU or display (or both) went idle.  Check the busy status
6675  * here and adjust the CRTC and GPU clocks as necessary.
6676  */
6677 static void intel_idle_update(struct work_struct *work)
6678 {
6679         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
6680                                                     idle_work);
6681         struct drm_device *dev = dev_priv->dev;
6682         struct drm_crtc *crtc;
6683         struct intel_crtc *intel_crtc;
6684
6685         if (!i915_powersave)
6686                 return;
6687
6688         mutex_lock(&dev->struct_mutex);
6689
6690         i915_update_gfx_val(dev_priv);
6691
6692         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
6693                 /* Skip inactive CRTCs */
6694                 if (!crtc->fb)
6695                         continue;
6696
6697                 intel_crtc = to_intel_crtc(crtc);
6698                 if (!intel_crtc->busy)
6699                         intel_decrease_pllclock(crtc);
6700         }
6701
6702
6703         mutex_unlock(&dev->struct_mutex);
6704 }
6705
6706 /**
6707  * intel_mark_busy - mark the GPU and possibly the display busy
6708  * @dev: drm device
6709  * @obj: object we're operating on
6710  *
6711  * Callers can use this function to indicate that the GPU is busy processing
6712  * commands.  If @obj matches one of the CRTC objects (i.e. it's a scanout
6713  * buffer), we'll also mark the display as busy, so we know to increase its
6714  * clock frequency.
6715  */
6716 void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
6717 {
6718         drm_i915_private_t *dev_priv = dev->dev_private;
6719         struct drm_crtc *crtc = NULL;
6720         struct intel_framebuffer *intel_fb;
6721         struct intel_crtc *intel_crtc;
6722
6723         if (!drm_core_check_feature(dev, DRIVER_MODESET))
6724                 return;
6725
6726         if (!dev_priv->busy)
6727                 dev_priv->busy = true;
6728         else
6729                 mod_timer(&dev_priv->idle_timer, jiffies +
6730                           msecs_to_jiffies(GPU_IDLE_TIMEOUT));
6731
6732         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
6733                 if (!crtc->fb)
6734                         continue;
6735
6736                 intel_crtc = to_intel_crtc(crtc);
6737                 intel_fb = to_intel_framebuffer(crtc->fb);
6738                 if (intel_fb->obj == obj) {
6739                         if (!intel_crtc->busy) {
6740                                 /* Non-busy -> busy, upclock */
6741                                 intel_increase_pllclock(crtc);
6742                                 intel_crtc->busy = true;
6743                         } else {
6744                                 /* Busy -> busy, put off timer */
6745                                 mod_timer(&intel_crtc->idle_timer, jiffies +
6746                                           msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
6747                         }
6748                 }
6749         }
6750 }
6751
6752 static void intel_crtc_destroy(struct drm_crtc *crtc)
6753 {
6754         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6755         struct drm_device *dev = crtc->dev;
6756         struct intel_unpin_work *work;
6757         unsigned long flags;
6758
6759         spin_lock_irqsave(&dev->event_lock, flags);
6760         work = intel_crtc->unpin_work;
6761         intel_crtc->unpin_work = NULL;
6762         spin_unlock_irqrestore(&dev->event_lock, flags);
6763
6764         if (work) {
6765                 cancel_work_sync(&work->work);
6766                 kfree(work);
6767         }
6768
6769         drm_crtc_cleanup(crtc);
6770
6771         kfree(intel_crtc);
6772 }
6773
6774 static void intel_unpin_work_fn(struct work_struct *__work)
6775 {
6776         struct intel_unpin_work *work =
6777                 container_of(__work, struct intel_unpin_work, work);
6778
6779         mutex_lock(&work->dev->struct_mutex);
6780         i915_gem_object_unpin(work->old_fb_obj);
6781         drm_gem_object_unreference(&work->pending_flip_obj->base);
6782         drm_gem_object_unreference(&work->old_fb_obj->base);
6783
6784         intel_update_fbc(work->dev);
6785         mutex_unlock(&work->dev->struct_mutex);
6786         kfree(work);
6787 }
6788
6789 static void do_intel_finish_page_flip(struct drm_device *dev,
6790                                       struct drm_crtc *crtc)
6791 {
6792         drm_i915_private_t *dev_priv = dev->dev_private;
6793         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6794         struct intel_unpin_work *work;
6795         struct drm_i915_gem_object *obj;
6796         struct drm_pending_vblank_event *e;
6797         struct timeval tnow, tvbl;
6798         unsigned long flags;
6799
6800         /* Ignore early vblank irqs */
6801         if (intel_crtc == NULL)
6802                 return;
6803
6804         do_gettimeofday(&tnow);
6805
6806         spin_lock_irqsave(&dev->event_lock, flags);
6807         work = intel_crtc->unpin_work;
6808         if (work == NULL || !work->pending) {
6809                 spin_unlock_irqrestore(&dev->event_lock, flags);
6810                 return;
6811         }
6812
6813         intel_crtc->unpin_work = NULL;
6814
6815         if (work->event) {
6816                 e = work->event;
6817                 e->event.sequence = drm_vblank_count_and_time(dev, intel_crtc->pipe, &tvbl);
6818
6819                 /* Called before vblank count and timestamps have
6820                  * been updated for the vblank interval of flip
6821                  * completion? Need to increment vblank count and
6822                  * add one videorefresh duration to returned timestamp
6823                  * to account for this. We assume this happened if we
6824                  * get called over 0.9 frame durations after the last
6825                  * timestamped vblank.
6826                  *
6827                  * This calculation can not be used with vrefresh rates
6828                  * below 5Hz (10Hz to be on the safe side) without
6829                  * promoting to 64 integers.
6830                  */
6831                 if (10 * (timeval_to_ns(&tnow) - timeval_to_ns(&tvbl)) >
6832                     9 * crtc->framedur_ns) {
6833                         e->event.sequence++;
6834                         tvbl = ns_to_timeval(timeval_to_ns(&tvbl) +
6835                                              crtc->framedur_ns);
6836                 }
6837
6838                 e->event.tv_sec = tvbl.tv_sec;
6839                 e->event.tv_usec = tvbl.tv_usec;
6840
6841                 list_add_tail(&e->base.link,
6842                               &e->base.file_priv->event_list);
6843                 wake_up_interruptible(&e->base.file_priv->event_wait);
6844         }
6845
6846         drm_vblank_put(dev, intel_crtc->pipe);
6847
6848         spin_unlock_irqrestore(&dev->event_lock, flags);
6849
6850         obj = work->old_fb_obj;
6851
6852         atomic_clear_mask(1 << intel_crtc->plane,
6853                           &obj->pending_flip.counter);
6854         if (atomic_read(&obj->pending_flip) == 0)
6855                 wake_up(&dev_priv->pending_flip_queue);
6856
6857         schedule_work(&work->work);
6858
6859         trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj);
6860 }
6861
6862 void intel_finish_page_flip(struct drm_device *dev, int pipe)
6863 {
6864         drm_i915_private_t *dev_priv = dev->dev_private;
6865         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
6866
6867         do_intel_finish_page_flip(dev, crtc);
6868 }
6869
6870 void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
6871 {
6872         drm_i915_private_t *dev_priv = dev->dev_private;
6873         struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane];
6874
6875         do_intel_finish_page_flip(dev, crtc);
6876 }
6877
6878 void intel_prepare_page_flip(struct drm_device *dev, int plane)
6879 {
6880         drm_i915_private_t *dev_priv = dev->dev_private;
6881         struct intel_crtc *intel_crtc =
6882                 to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
6883         unsigned long flags;
6884
6885         spin_lock_irqsave(&dev->event_lock, flags);
6886         if (intel_crtc->unpin_work) {
6887                 if ((++intel_crtc->unpin_work->pending) > 1)
6888                         DRM_ERROR("Prepared flip multiple times\n");
6889         } else {
6890                 DRM_DEBUG_DRIVER("preparing flip with no unpin work?\n");
6891         }
6892         spin_unlock_irqrestore(&dev->event_lock, flags);
6893 }
6894
6895 static int intel_gen2_queue_flip(struct drm_device *dev,
6896                                  struct drm_crtc *crtc,
6897                                  struct drm_framebuffer *fb,
6898                                  struct drm_i915_gem_object *obj)
6899 {
6900         struct drm_i915_private *dev_priv = dev->dev_private;
6901         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6902         unsigned long offset;
6903         u32 flip_mask;
6904         int ret;
6905
6906         ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
6907         if (ret)
6908                 goto out;
6909
6910         /* Offset into the new buffer for cases of shared fbs between CRTCs */
6911         offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8;
6912
6913         ret = BEGIN_LP_RING(6);
6914         if (ret)
6915                 goto out;
6916
6917         /* Can't queue multiple flips, so wait for the previous
6918          * one to finish before executing the next.
6919          */
6920         if (intel_crtc->plane)
6921                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
6922         else
6923                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
6924         OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
6925         OUT_RING(MI_NOOP);
6926         OUT_RING(MI_DISPLAY_FLIP |
6927                  MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
6928         OUT_RING(fb->pitch);
6929         OUT_RING(obj->gtt_offset + offset);
6930         OUT_RING(MI_NOOP);
6931         ADVANCE_LP_RING();
6932 out:
6933         return ret;
6934 }
6935
6936 static int intel_gen3_queue_flip(struct drm_device *dev,
6937                                  struct drm_crtc *crtc,
6938                                  struct drm_framebuffer *fb,
6939                                  struct drm_i915_gem_object *obj)
6940 {
6941         struct drm_i915_private *dev_priv = dev->dev_private;
6942         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6943         unsigned long offset;
6944         u32 flip_mask;
6945         int ret;
6946
6947         ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
6948         if (ret)
6949                 goto out;
6950
6951         /* Offset into the new buffer for cases of shared fbs between CRTCs */
6952         offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8;
6953
6954         ret = BEGIN_LP_RING(6);
6955         if (ret)
6956                 goto out;
6957
6958         if (intel_crtc->plane)
6959                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
6960         else
6961                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
6962         OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
6963         OUT_RING(MI_NOOP);
6964         OUT_RING(MI_DISPLAY_FLIP_I915 |
6965                  MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
6966         OUT_RING(fb->pitch);
6967         OUT_RING(obj->gtt_offset + offset);
6968         OUT_RING(MI_NOOP);
6969
6970         ADVANCE_LP_RING();
6971 out:
6972         return ret;
6973 }
6974
6975 static int intel_gen4_queue_flip(struct drm_device *dev,
6976                                  struct drm_crtc *crtc,
6977                                  struct drm_framebuffer *fb,
6978                                  struct drm_i915_gem_object *obj)
6979 {
6980         struct drm_i915_private *dev_priv = dev->dev_private;
6981         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6982         uint32_t pf, pipesrc;
6983         int ret;
6984
6985         ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
6986         if (ret)
6987                 goto out;
6988
6989         ret = BEGIN_LP_RING(4);
6990         if (ret)
6991                 goto out;
6992
6993         /* i965+ uses the linear or tiled offsets from the
6994          * Display Registers (which do not change across a page-flip)
6995          * so we need only reprogram the base address.
6996          */
6997         OUT_RING(MI_DISPLAY_FLIP |
6998                  MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
6999         OUT_RING(fb->pitch);
7000         OUT_RING(obj->gtt_offset | obj->tiling_mode);
7001
7002         /* XXX Enabling the panel-fitter across page-flip is so far
7003          * untested on non-native modes, so ignore it for now.
7004          * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
7005          */
7006         pf = 0;
7007         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
7008         OUT_RING(pf | pipesrc);
7009         ADVANCE_LP_RING();
7010 out:
7011         return ret;
7012 }
7013
7014 static int intel_gen6_queue_flip(struct drm_device *dev,
7015                                  struct drm_crtc *crtc,
7016                                  struct drm_framebuffer *fb,
7017                                  struct drm_i915_gem_object *obj)
7018 {
7019         struct drm_i915_private *dev_priv = dev->dev_private;
7020         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7021         uint32_t pf, pipesrc;
7022         int ret;
7023
7024         ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
7025         if (ret)
7026                 goto out;
7027
7028         ret = BEGIN_LP_RING(4);
7029         if (ret)
7030                 goto out;
7031
7032         OUT_RING(MI_DISPLAY_FLIP |
7033                  MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
7034         OUT_RING(fb->pitch | obj->tiling_mode);
7035         OUT_RING(obj->gtt_offset);
7036
7037         pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
7038         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
7039         OUT_RING(pf | pipesrc);
7040         ADVANCE_LP_RING();
7041 out:
7042         return ret;
7043 }
7044
7045 /*
7046  * On gen7 we currently use the blit ring because (in early silicon at least)
7047  * the render ring doesn't give us interrpts for page flip completion, which
7048  * means clients will hang after the first flip is queued.  Fortunately the
7049  * blit ring generates interrupts properly, so use it instead.
7050  */
7051 static int intel_gen7_queue_flip(struct drm_device *dev,
7052                                  struct drm_crtc *crtc,
7053                                  struct drm_framebuffer *fb,
7054                                  struct drm_i915_gem_object *obj)
7055 {
7056         struct drm_i915_private *dev_priv = dev->dev_private;
7057         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7058         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
7059         int ret;
7060
7061         ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
7062         if (ret)
7063                 goto out;
7064
7065         ret = intel_ring_begin(ring, 4);
7066         if (ret)
7067                 goto out;
7068
7069         intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | (intel_crtc->plane << 19));
7070         intel_ring_emit(ring, (fb->pitch | obj->tiling_mode));
7071         intel_ring_emit(ring, (obj->gtt_offset));
7072         intel_ring_emit(ring, (MI_NOOP));
7073         intel_ring_advance(ring);
7074 out:
7075         return ret;
7076 }
7077
7078 static int intel_default_queue_flip(struct drm_device *dev,
7079                                     struct drm_crtc *crtc,
7080                                     struct drm_framebuffer *fb,
7081                                     struct drm_i915_gem_object *obj)
7082 {
7083         return -ENODEV;
7084 }
7085
7086 static int intel_crtc_page_flip(struct drm_crtc *crtc,
7087                                 struct drm_framebuffer *fb,
7088                                 struct drm_pending_vblank_event *event)
7089 {
7090         struct drm_device *dev = crtc->dev;
7091         struct drm_i915_private *dev_priv = dev->dev_private;
7092         struct intel_framebuffer *intel_fb;
7093         struct drm_i915_gem_object *obj;
7094         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7095         struct intel_unpin_work *work;
7096         unsigned long flags;
7097         int ret;
7098
7099         work = kzalloc(sizeof *work, GFP_KERNEL);
7100         if (work == NULL)
7101                 return -ENOMEM;
7102
7103         work->event = event;
7104         work->dev = crtc->dev;
7105         intel_fb = to_intel_framebuffer(crtc->fb);
7106         work->old_fb_obj = intel_fb->obj;
7107         INIT_WORK(&work->work, intel_unpin_work_fn);
7108
7109         /* We borrow the event spin lock for protecting unpin_work */
7110         spin_lock_irqsave(&dev->event_lock, flags);
7111         if (intel_crtc->unpin_work) {
7112                 spin_unlock_irqrestore(&dev->event_lock, flags);
7113                 kfree(work);
7114
7115                 DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
7116                 return -EBUSY;
7117         }
7118         intel_crtc->unpin_work = work;
7119         spin_unlock_irqrestore(&dev->event_lock, flags);
7120
7121         intel_fb = to_intel_framebuffer(fb);
7122         obj = intel_fb->obj;
7123
7124         mutex_lock(&dev->struct_mutex);
7125
7126         /* Reference the objects for the scheduled work. */
7127         drm_gem_object_reference(&work->old_fb_obj->base);
7128         drm_gem_object_reference(&obj->base);
7129
7130         crtc->fb = fb;
7131
7132         ret = drm_vblank_get(dev, intel_crtc->pipe);
7133         if (ret)
7134                 goto cleanup_objs;
7135
7136         work->pending_flip_obj = obj;
7137
7138         work->enable_stall_check = true;
7139
7140         /* Block clients from rendering to the new back buffer until
7141          * the flip occurs and the object is no longer visible.
7142          */
7143         atomic_add(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
7144
7145         ret = dev_priv->display.queue_flip(dev, crtc, fb, obj);
7146         if (ret)
7147                 goto cleanup_pending;
7148
7149         intel_disable_fbc(dev);
7150         mutex_unlock(&dev->struct_mutex);
7151
7152         trace_i915_flip_request(intel_crtc->plane, obj);
7153
7154         return 0;
7155
7156 cleanup_pending:
7157         atomic_sub(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
7158 cleanup_objs:
7159         drm_gem_object_unreference(&work->old_fb_obj->base);
7160         drm_gem_object_unreference(&obj->base);
7161         mutex_unlock(&dev->struct_mutex);
7162
7163         spin_lock_irqsave(&dev->event_lock, flags);
7164         intel_crtc->unpin_work = NULL;
7165         spin_unlock_irqrestore(&dev->event_lock, flags);
7166
7167         kfree(work);
7168
7169         return ret;
7170 }
7171
7172 static void intel_sanitize_modesetting(struct drm_device *dev,
7173                                        int pipe, int plane)
7174 {
7175         struct drm_i915_private *dev_priv = dev->dev_private;
7176         u32 reg, val;
7177
7178         if (HAS_PCH_SPLIT(dev))
7179                 return;
7180
7181         /* Who knows what state these registers were left in by the BIOS or
7182          * grub?
7183          *
7184          * If we leave the registers in a conflicting state (e.g. with the
7185          * display plane reading from the other pipe than the one we intend
7186          * to use) then when we attempt to teardown the active mode, we will
7187          * not disable the pipes and planes in the correct order -- leaving
7188          * a plane reading from a disabled pipe and possibly leading to
7189          * undefined behaviour.
7190          */
7191
7192         reg = DSPCNTR(plane);
7193         val = I915_READ(reg);
7194
7195         if ((val & DISPLAY_PLANE_ENABLE) == 0)
7196                 return;
7197         if (!!(val & DISPPLANE_SEL_PIPE_MASK) == pipe)
7198                 return;
7199
7200         /* This display plane is active and attached to the other CPU pipe. */
7201         pipe = !pipe;
7202
7203         /* Disable the plane and wait for it to stop reading from the pipe. */
7204         intel_disable_plane(dev_priv, plane, pipe);
7205         intel_disable_pipe(dev_priv, pipe);
7206 }
7207
7208 static void intel_crtc_reset(struct drm_crtc *crtc)
7209 {
7210         struct drm_device *dev = crtc->dev;
7211         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7212
7213         /* Reset flags back to the 'unknown' status so that they
7214          * will be correctly set on the initial modeset.
7215          */
7216         intel_crtc->dpms_mode = -1;
7217
7218         /* We need to fix up any BIOS configuration that conflicts with
7219          * our expectations.
7220          */
7221         intel_sanitize_modesetting(dev, intel_crtc->pipe, intel_crtc->plane);
7222 }
7223
7224 static struct drm_crtc_helper_funcs intel_helper_funcs = {
7225         .dpms = intel_crtc_dpms,
7226         .mode_fixup = intel_crtc_mode_fixup,
7227         .mode_set = intel_crtc_mode_set,
7228         .mode_set_base = intel_pipe_set_base,
7229         .mode_set_base_atomic = intel_pipe_set_base_atomic,
7230         .load_lut = intel_crtc_load_lut,
7231         .disable = intel_crtc_disable,
7232 };
7233
7234 static const struct drm_crtc_funcs intel_crtc_funcs = {
7235         .reset = intel_crtc_reset,
7236         .cursor_set = intel_crtc_cursor_set,
7237         .cursor_move = intel_crtc_cursor_move,
7238         .gamma_set = intel_crtc_gamma_set,
7239         .set_config = drm_crtc_helper_set_config,
7240         .destroy = intel_crtc_destroy,
7241         .page_flip = intel_crtc_page_flip,
7242 };
7243
7244 static void intel_crtc_init(struct drm_device *dev, int pipe)
7245 {
7246         drm_i915_private_t *dev_priv = dev->dev_private;
7247         struct intel_crtc *intel_crtc;
7248         int i;
7249
7250         intel_crtc = kzalloc(sizeof(struct intel_crtc) + (INTELFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
7251         if (intel_crtc == NULL)
7252                 return;
7253
7254         drm_crtc_init(dev, &intel_crtc->base, &intel_crtc_funcs);
7255
7256         drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
7257         for (i = 0; i < 256; i++) {
7258                 intel_crtc->lut_r[i] = i;
7259                 intel_crtc->lut_g[i] = i;
7260                 intel_crtc->lut_b[i] = i;
7261         }
7262
7263         /* Swap pipes & planes for FBC on pre-965 */
7264         intel_crtc->pipe = pipe;
7265         intel_crtc->plane = pipe;
7266         if (IS_MOBILE(dev) && IS_GEN3(dev)) {
7267                 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
7268                 intel_crtc->plane = !pipe;
7269         }
7270
7271         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
7272                dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
7273         dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
7274         dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
7275
7276         intel_crtc_reset(&intel_crtc->base);
7277         intel_crtc->active = true; /* force the pipe off on setup_init_config */
7278         intel_crtc->bpp = 24; /* default for pre-Ironlake */
7279
7280         if (HAS_PCH_SPLIT(dev)) {
7281                 intel_helper_funcs.prepare = ironlake_crtc_prepare;
7282                 intel_helper_funcs.commit = ironlake_crtc_commit;
7283         } else {
7284                 intel_helper_funcs.prepare = i9xx_crtc_prepare;
7285                 intel_helper_funcs.commit = i9xx_crtc_commit;
7286         }
7287
7288         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
7289
7290         intel_crtc->busy = false;
7291
7292         setup_timer(&intel_crtc->idle_timer, intel_crtc_idle_timer,
7293                     (unsigned long)intel_crtc);
7294 }
7295
7296 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
7297                                 struct drm_file *file)
7298 {
7299         drm_i915_private_t *dev_priv = dev->dev_private;
7300         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
7301         struct drm_mode_object *drmmode_obj;
7302         struct intel_crtc *crtc;
7303
7304         if (!dev_priv) {
7305                 DRM_ERROR("called with no initialization\n");
7306                 return -EINVAL;
7307         }
7308
7309         drmmode_obj = drm_mode_object_find(dev, pipe_from_crtc_id->crtc_id,
7310                         DRM_MODE_OBJECT_CRTC);
7311
7312         if (!drmmode_obj) {
7313                 DRM_ERROR("no such CRTC id\n");
7314                 return -EINVAL;
7315         }
7316
7317         crtc = to_intel_crtc(obj_to_crtc(drmmode_obj));
7318         pipe_from_crtc_id->pipe = crtc->pipe;
7319
7320         return 0;
7321 }
7322
7323 static int intel_encoder_clones(struct drm_device *dev, int type_mask)
7324 {
7325         struct intel_encoder *encoder;
7326         int index_mask = 0;
7327         int entry = 0;
7328
7329         list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
7330                 if (type_mask & encoder->clone_mask)
7331                         index_mask |= (1 << entry);
7332                 entry++;
7333         }
7334
7335         return index_mask;
7336 }
7337
7338 static bool has_edp_a(struct drm_device *dev)
7339 {
7340         struct drm_i915_private *dev_priv = dev->dev_private;
7341
7342         if (!IS_MOBILE(dev))
7343                 return false;
7344
7345         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
7346                 return false;
7347
7348         if (IS_GEN5(dev) &&
7349             (I915_READ(ILK_DISPLAY_CHICKEN_FUSES) & ILK_eDP_A_DISABLE))
7350                 return false;
7351
7352         return true;
7353 }
7354
7355 static void intel_setup_outputs(struct drm_device *dev)
7356 {
7357         struct drm_i915_private *dev_priv = dev->dev_private;
7358         struct intel_encoder *encoder;
7359         bool dpd_is_edp = false;
7360         bool has_lvds = false;
7361
7362         if (IS_MOBILE(dev) && !IS_I830(dev))
7363                 has_lvds = intel_lvds_init(dev);
7364         if (!has_lvds && !HAS_PCH_SPLIT(dev)) {
7365                 /* disable the panel fitter on everything but LVDS */
7366                 I915_WRITE(PFIT_CONTROL, 0);
7367         }
7368
7369         if (HAS_PCH_SPLIT(dev)) {
7370                 dpd_is_edp = intel_dpd_is_edp(dev);
7371
7372                 if (has_edp_a(dev))
7373                         intel_dp_init(dev, DP_A);
7374
7375                 if (dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
7376                         intel_dp_init(dev, PCH_DP_D);
7377         }
7378
7379         intel_crt_init(dev);
7380
7381         if (HAS_PCH_SPLIT(dev)) {
7382                 int found;
7383
7384                 if (I915_READ(HDMIB) & PORT_DETECTED) {
7385                         /* PCH SDVOB multiplex with HDMIB */
7386                         found = intel_sdvo_init(dev, PCH_SDVOB);
7387                         if (!found)
7388                                 intel_hdmi_init(dev, HDMIB);
7389                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
7390                                 intel_dp_init(dev, PCH_DP_B);
7391                 }
7392
7393                 if (I915_READ(HDMIC) & PORT_DETECTED)
7394                         intel_hdmi_init(dev, HDMIC);
7395
7396                 if (I915_READ(HDMID) & PORT_DETECTED)
7397                         intel_hdmi_init(dev, HDMID);
7398
7399                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
7400                         intel_dp_init(dev, PCH_DP_C);
7401
7402                 if (!dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
7403                         intel_dp_init(dev, PCH_DP_D);
7404
7405         } else if (SUPPORTS_DIGITAL_OUTPUTS(dev)) {
7406                 bool found = false;
7407
7408                 if (I915_READ(SDVOB) & SDVO_DETECTED) {
7409                         DRM_DEBUG_KMS("probing SDVOB\n");
7410                         found = intel_sdvo_init(dev, SDVOB);
7411                         if (!found && SUPPORTS_INTEGRATED_HDMI(dev)) {
7412                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
7413                                 intel_hdmi_init(dev, SDVOB);
7414                         }
7415
7416                         if (!found && SUPPORTS_INTEGRATED_DP(dev)) {
7417                                 DRM_DEBUG_KMS("probing DP_B\n");
7418                                 intel_dp_init(dev, DP_B);
7419                         }
7420                 }
7421
7422                 /* Before G4X SDVOC doesn't have its own detect register */
7423
7424                 if (I915_READ(SDVOB) & SDVO_DETECTED) {
7425                         DRM_DEBUG_KMS("probing SDVOC\n");
7426                         found = intel_sdvo_init(dev, SDVOC);
7427                 }
7428
7429                 if (!found && (I915_READ(SDVOC) & SDVO_DETECTED)) {
7430
7431                         if (SUPPORTS_INTEGRATED_HDMI(dev)) {
7432                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
7433                                 intel_hdmi_init(dev, SDVOC);
7434                         }
7435                         if (SUPPORTS_INTEGRATED_DP(dev)) {
7436                                 DRM_DEBUG_KMS("probing DP_C\n");
7437                                 intel_dp_init(dev, DP_C);
7438                         }
7439                 }
7440
7441                 if (SUPPORTS_INTEGRATED_DP(dev) &&
7442                     (I915_READ(DP_D) & DP_DETECTED)) {
7443                         DRM_DEBUG_KMS("probing DP_D\n");
7444                         intel_dp_init(dev, DP_D);
7445                 }
7446         } else if (IS_GEN2(dev))
7447                 intel_dvo_init(dev);
7448
7449         if (SUPPORTS_TV(dev))
7450                 intel_tv_init(dev);
7451
7452         list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
7453                 encoder->base.possible_crtcs = encoder->crtc_mask;
7454                 encoder->base.possible_clones =
7455                         intel_encoder_clones(dev, encoder->clone_mask);
7456         }
7457
7458         /* disable all the possible outputs/crtcs before entering KMS mode */
7459         drm_helper_disable_unused_functions(dev);
7460
7461         if (HAS_PCH_SPLIT(dev))
7462                 ironlake_init_pch_refclk(dev);
7463 }
7464
7465 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
7466 {
7467         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
7468
7469         drm_framebuffer_cleanup(fb);
7470         drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
7471
7472         kfree(intel_fb);
7473 }
7474
7475 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
7476                                                 struct drm_file *file,
7477                                                 unsigned int *handle)
7478 {
7479         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
7480         struct drm_i915_gem_object *obj = intel_fb->obj;
7481
7482         return drm_gem_handle_create(file, &obj->base, handle);
7483 }
7484
7485 static const struct drm_framebuffer_funcs intel_fb_funcs = {
7486         .destroy = intel_user_framebuffer_destroy,
7487         .create_handle = intel_user_framebuffer_create_handle,
7488 };
7489
7490 int intel_framebuffer_init(struct drm_device *dev,
7491                            struct intel_framebuffer *intel_fb,
7492                            struct drm_mode_fb_cmd *mode_cmd,
7493                            struct drm_i915_gem_object *obj)
7494 {
7495         int ret;
7496
7497         if (obj->tiling_mode == I915_TILING_Y)
7498                 return -EINVAL;
7499
7500         if (mode_cmd->pitch & 63)
7501                 return -EINVAL;
7502
7503         switch (mode_cmd->bpp) {
7504         case 8:
7505         case 16:
7506                 /* Only pre-ILK can handle 5:5:5 */
7507                 if (mode_cmd->depth == 15 && !HAS_PCH_SPLIT(dev))
7508                         return -EINVAL;
7509                 break;
7510
7511         case 24:
7512         case 32:
7513                 break;
7514         default:
7515                 return -EINVAL;
7516         }
7517
7518         ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
7519         if (ret) {
7520                 DRM_ERROR("framebuffer init failed %d\n", ret);
7521                 return ret;
7522         }
7523
7524         drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd);
7525         intel_fb->obj = obj;
7526         return 0;
7527 }
7528
7529 static struct drm_framebuffer *
7530 intel_user_framebuffer_create(struct drm_device *dev,
7531                               struct drm_file *filp,
7532                               struct drm_mode_fb_cmd *mode_cmd)
7533 {
7534         struct drm_i915_gem_object *obj;
7535
7536         obj = to_intel_bo(drm_gem_object_lookup(dev, filp, mode_cmd->handle));
7537         if (&obj->base == NULL)
7538                 return ERR_PTR(-ENOENT);
7539
7540         return intel_framebuffer_create(dev, mode_cmd, obj);
7541 }
7542
7543 static const struct drm_mode_config_funcs intel_mode_funcs = {
7544         .fb_create = intel_user_framebuffer_create,
7545         .output_poll_changed = intel_fb_output_poll_changed,
7546 };
7547
7548 static struct drm_i915_gem_object *
7549 intel_alloc_context_page(struct drm_device *dev)
7550 {
7551         struct drm_i915_gem_object *ctx;
7552         int ret;
7553
7554         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
7555
7556         ctx = i915_gem_alloc_object(dev, 4096);
7557         if (!ctx) {
7558                 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
7559                 return NULL;
7560         }
7561
7562         ret = i915_gem_object_pin(ctx, 4096, true);
7563         if (ret) {
7564                 DRM_ERROR("failed to pin power context: %d\n", ret);
7565                 goto err_unref;
7566         }
7567
7568         ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
7569         if (ret) {
7570                 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
7571                 goto err_unpin;
7572         }
7573
7574         return ctx;
7575
7576 err_unpin:
7577         i915_gem_object_unpin(ctx);
7578 err_unref:
7579         drm_gem_object_unreference(&ctx->base);
7580         mutex_unlock(&dev->struct_mutex);
7581         return NULL;
7582 }
7583
7584 bool ironlake_set_drps(struct drm_device *dev, u8 val)
7585 {
7586         struct drm_i915_private *dev_priv = dev->dev_private;
7587         u16 rgvswctl;
7588
7589         rgvswctl = I915_READ16(MEMSWCTL);
7590         if (rgvswctl & MEMCTL_CMD_STS) {
7591                 DRM_DEBUG("gpu busy, RCS change rejected\n");
7592                 return false; /* still busy with another command */
7593         }
7594
7595         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
7596                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
7597         I915_WRITE16(MEMSWCTL, rgvswctl);
7598         POSTING_READ16(MEMSWCTL);
7599
7600         rgvswctl |= MEMCTL_CMD_STS;
7601         I915_WRITE16(MEMSWCTL, rgvswctl);
7602
7603         return true;
7604 }
7605
7606 void ironlake_enable_drps(struct drm_device *dev)
7607 {
7608         struct drm_i915_private *dev_priv = dev->dev_private;
7609         u32 rgvmodectl = I915_READ(MEMMODECTL);
7610         u8 fmax, fmin, fstart, vstart;
7611
7612         /* Enable temp reporting */
7613         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
7614         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
7615
7616         /* 100ms RC evaluation intervals */
7617         I915_WRITE(RCUPEI, 100000);
7618         I915_WRITE(RCDNEI, 100000);
7619
7620         /* Set max/min thresholds to 90ms and 80ms respectively */
7621         I915_WRITE(RCBMAXAVG, 90000);
7622         I915_WRITE(RCBMINAVG, 80000);
7623
7624         I915_WRITE(MEMIHYST, 1);
7625
7626         /* Set up min, max, and cur for interrupt handling */
7627         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
7628         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
7629         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
7630                 MEMMODE_FSTART_SHIFT;
7631
7632         vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
7633                 PXVFREQ_PX_SHIFT;
7634
7635         dev_priv->fmax = fmax; /* IPS callback will increase this */
7636         dev_priv->fstart = fstart;
7637
7638         dev_priv->max_delay = fstart;
7639         dev_priv->min_delay = fmin;
7640         dev_priv->cur_delay = fstart;
7641
7642         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
7643                          fmax, fmin, fstart);
7644
7645         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
7646
7647         /*
7648          * Interrupts will be enabled in ironlake_irq_postinstall
7649          */
7650
7651         I915_WRITE(VIDSTART, vstart);
7652         POSTING_READ(VIDSTART);
7653
7654         rgvmodectl |= MEMMODE_SWMODE_EN;
7655         I915_WRITE(MEMMODECTL, rgvmodectl);
7656
7657         if (wait_for((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
7658                 DRM_ERROR("stuck trying to change perf mode\n");
7659         msleep(1);
7660
7661         ironlake_set_drps(dev, fstart);
7662
7663         dev_priv->last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
7664                 I915_READ(0x112e0);
7665         dev_priv->last_time1 = jiffies_to_msecs(jiffies);
7666         dev_priv->last_count2 = I915_READ(0x112f4);
7667         getrawmonotonic(&dev_priv->last_time2);
7668 }
7669
7670 void ironlake_disable_drps(struct drm_device *dev)
7671 {
7672         struct drm_i915_private *dev_priv = dev->dev_private;
7673         u16 rgvswctl = I915_READ16(MEMSWCTL);
7674
7675         /* Ack interrupts, disable EFC interrupt */
7676         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
7677         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
7678         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
7679         I915_WRITE(DEIIR, DE_PCU_EVENT);
7680         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
7681
7682         /* Go back to the starting frequency */
7683         ironlake_set_drps(dev, dev_priv->fstart);
7684         msleep(1);
7685         rgvswctl |= MEMCTL_CMD_STS;
7686         I915_WRITE(MEMSWCTL, rgvswctl);
7687         msleep(1);
7688
7689 }
7690
7691 void gen6_set_rps(struct drm_device *dev, u8 val)
7692 {
7693         struct drm_i915_private *dev_priv = dev->dev_private;
7694         u32 swreq;
7695
7696         swreq = (val & 0x3ff) << 25;
7697         I915_WRITE(GEN6_RPNSWREQ, swreq);
7698 }
7699
7700 void gen6_disable_rps(struct drm_device *dev)
7701 {
7702         struct drm_i915_private *dev_priv = dev->dev_private;
7703
7704         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
7705         I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
7706         I915_WRITE(GEN6_PMIER, 0);
7707         /* Complete PM interrupt masking here doesn't race with the rps work
7708          * item again unmasking PM interrupts because that is using a different
7709          * register (PMIMR) to mask PM interrupts. The only risk is in leaving
7710          * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
7711
7712         spin_lock_irq(&dev_priv->rps_lock);
7713         dev_priv->pm_iir = 0;
7714         spin_unlock_irq(&dev_priv->rps_lock);
7715
7716         I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
7717 }
7718
7719 static unsigned long intel_pxfreq(u32 vidfreq)
7720 {
7721         unsigned long freq;
7722         int div = (vidfreq & 0x3f0000) >> 16;
7723         int post = (vidfreq & 0x3000) >> 12;
7724         int pre = (vidfreq & 0x7);
7725
7726         if (!pre)
7727                 return 0;
7728
7729         freq = ((div * 133333) / ((1<<post) * pre));
7730
7731         return freq;
7732 }
7733
7734 void intel_init_emon(struct drm_device *dev)
7735 {
7736         struct drm_i915_private *dev_priv = dev->dev_private;
7737         u32 lcfuse;
7738         u8 pxw[16];
7739         int i;
7740
7741         /* Disable to program */
7742         I915_WRITE(ECR, 0);
7743         POSTING_READ(ECR);
7744
7745         /* Program energy weights for various events */
7746         I915_WRITE(SDEW, 0x15040d00);
7747         I915_WRITE(CSIEW0, 0x007f0000);
7748         I915_WRITE(CSIEW1, 0x1e220004);
7749         I915_WRITE(CSIEW2, 0x04000004);
7750
7751         for (i = 0; i < 5; i++)
7752                 I915_WRITE(PEW + (i * 4), 0);
7753         for (i = 0; i < 3; i++)
7754                 I915_WRITE(DEW + (i * 4), 0);
7755
7756         /* Program P-state weights to account for frequency power adjustment */
7757         for (i = 0; i < 16; i++) {
7758                 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
7759                 unsigned long freq = intel_pxfreq(pxvidfreq);
7760                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
7761                         PXVFREQ_PX_SHIFT;
7762                 unsigned long val;
7763
7764                 val = vid * vid;
7765                 val *= (freq / 1000);
7766                 val *= 255;
7767                 val /= (127*127*900);
7768                 if (val > 0xff)
7769                         DRM_ERROR("bad pxval: %ld\n", val);
7770                 pxw[i] = val;
7771         }
7772         /* Render standby states get 0 weight */
7773         pxw[14] = 0;
7774         pxw[15] = 0;
7775
7776         for (i = 0; i < 4; i++) {
7777                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
7778                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
7779                 I915_WRITE(PXW + (i * 4), val);
7780         }
7781
7782         /* Adjust magic regs to magic values (more experimental results) */
7783         I915_WRITE(OGW0, 0);
7784         I915_WRITE(OGW1, 0);
7785         I915_WRITE(EG0, 0x00007f00);
7786         I915_WRITE(EG1, 0x0000000e);
7787         I915_WRITE(EG2, 0x000e0000);
7788         I915_WRITE(EG3, 0x68000300);
7789         I915_WRITE(EG4, 0x42000000);
7790         I915_WRITE(EG5, 0x00140031);
7791         I915_WRITE(EG6, 0);
7792         I915_WRITE(EG7, 0);
7793
7794         for (i = 0; i < 8; i++)
7795                 I915_WRITE(PXWL + (i * 4), 0);
7796
7797         /* Enable PMON + select events */
7798         I915_WRITE(ECR, 0x80000019);
7799
7800         lcfuse = I915_READ(LCFUSE02);
7801
7802         dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
7803 }
7804
7805 void gen6_enable_rps(struct drm_i915_private *dev_priv)
7806 {
7807         u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
7808         u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
7809         u32 pcu_mbox, rc6_mask = 0;
7810         int cur_freq, min_freq, max_freq;
7811         int i;
7812
7813         /* Here begins a magic sequence of register writes to enable
7814          * auto-downclocking.
7815          *
7816          * Perhaps there might be some value in exposing these to
7817          * userspace...
7818          */
7819         I915_WRITE(GEN6_RC_STATE, 0);
7820         mutex_lock(&dev_priv->dev->struct_mutex);
7821         gen6_gt_force_wake_get(dev_priv);
7822
7823         /* disable the counters and set deterministic thresholds */
7824         I915_WRITE(GEN6_RC_CONTROL, 0);
7825
7826         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
7827         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
7828         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
7829         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7830         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7831
7832         for (i = 0; i < I915_NUM_RINGS; i++)
7833                 I915_WRITE(RING_MAX_IDLE(dev_priv->ring[i].mmio_base), 10);
7834
7835         I915_WRITE(GEN6_RC_SLEEP, 0);
7836         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
7837         I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7838         I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
7839         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
7840
7841         if (i915_enable_rc6)
7842                 rc6_mask = GEN6_RC_CTL_RC6p_ENABLE |
7843                         GEN6_RC_CTL_RC6_ENABLE;
7844
7845         I915_WRITE(GEN6_RC_CONTROL,
7846                    rc6_mask |
7847                    GEN6_RC_CTL_EI_MODE(1) |
7848                    GEN6_RC_CTL_HW_ENABLE);
7849
7850         I915_WRITE(GEN6_RPNSWREQ,
7851                    GEN6_FREQUENCY(10) |
7852                    GEN6_OFFSET(0) |
7853                    GEN6_AGGRESSIVE_TURBO);
7854         I915_WRITE(GEN6_RC_VIDEO_FREQ,
7855                    GEN6_FREQUENCY(12));
7856
7857         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7858         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
7859                    18 << 24 |
7860                    6 << 16);
7861         I915_WRITE(GEN6_RP_UP_THRESHOLD, 10000);
7862         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 1000000);
7863         I915_WRITE(GEN6_RP_UP_EI, 100000);
7864         I915_WRITE(GEN6_RP_DOWN_EI, 5000000);
7865         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7866         I915_WRITE(GEN6_RP_CONTROL,
7867                    GEN6_RP_MEDIA_TURBO |
7868                    GEN6_RP_USE_NORMAL_FREQ |
7869                    GEN6_RP_MEDIA_IS_GFX |
7870                    GEN6_RP_ENABLE |
7871                    GEN6_RP_UP_BUSY_AVG |
7872                    GEN6_RP_DOWN_IDLE_CONT);
7873
7874         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
7875                      500))
7876                 DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
7877
7878         I915_WRITE(GEN6_PCODE_DATA, 0);
7879         I915_WRITE(GEN6_PCODE_MAILBOX,
7880                    GEN6_PCODE_READY |
7881                    GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
7882         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
7883                      500))
7884                 DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
7885
7886         min_freq = (rp_state_cap & 0xff0000) >> 16;
7887         max_freq = rp_state_cap & 0xff;
7888         cur_freq = (gt_perf_status & 0xff00) >> 8;
7889
7890         /* Check for overclock support */
7891         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
7892                      500))
7893                 DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
7894         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
7895         pcu_mbox = I915_READ(GEN6_PCODE_DATA);
7896         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
7897                      500))
7898                 DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
7899         if (pcu_mbox & (1<<31)) { /* OC supported */
7900                 max_freq = pcu_mbox & 0xff;
7901                 DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
7902         }
7903
7904         /* In units of 100MHz */
7905         dev_priv->max_delay = max_freq;
7906         dev_priv->min_delay = min_freq;
7907         dev_priv->cur_delay = cur_freq;
7908
7909         /* requires MSI enabled */
7910         I915_WRITE(GEN6_PMIER,
7911                    GEN6_PM_MBOX_EVENT |
7912                    GEN6_PM_THERMAL_EVENT |
7913                    GEN6_PM_RP_DOWN_TIMEOUT |
7914                    GEN6_PM_RP_UP_THRESHOLD |
7915                    GEN6_PM_RP_DOWN_THRESHOLD |
7916                    GEN6_PM_RP_UP_EI_EXPIRED |
7917                    GEN6_PM_RP_DOWN_EI_EXPIRED);
7918         spin_lock_irq(&dev_priv->rps_lock);
7919         WARN_ON(dev_priv->pm_iir != 0);
7920         I915_WRITE(GEN6_PMIMR, 0);
7921         spin_unlock_irq(&dev_priv->rps_lock);
7922         /* enable all PM interrupts */
7923         I915_WRITE(GEN6_PMINTRMSK, 0);
7924
7925         gen6_gt_force_wake_put(dev_priv);
7926         mutex_unlock(&dev_priv->dev->struct_mutex);
7927 }
7928
7929 void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
7930 {
7931         int min_freq = 15;
7932         int gpu_freq, ia_freq, max_ia_freq;
7933         int scaling_factor = 180;
7934
7935         max_ia_freq = cpufreq_quick_get_max(0);
7936         /*
7937          * Default to measured freq if none found, PCU will ensure we don't go
7938          * over
7939          */
7940         if (!max_ia_freq)
7941                 max_ia_freq = tsc_khz;
7942
7943         /* Convert from kHz to MHz */
7944         max_ia_freq /= 1000;
7945
7946         mutex_lock(&dev_priv->dev->struct_mutex);
7947
7948         /*
7949          * For each potential GPU frequency, load a ring frequency we'd like
7950          * to use for memory access.  We do this by specifying the IA frequency
7951          * the PCU should use as a reference to determine the ring frequency.
7952          */
7953         for (gpu_freq = dev_priv->max_delay; gpu_freq >= dev_priv->min_delay;
7954              gpu_freq--) {
7955                 int diff = dev_priv->max_delay - gpu_freq;
7956
7957                 /*
7958                  * For GPU frequencies less than 750MHz, just use the lowest
7959                  * ring freq.
7960                  */
7961                 if (gpu_freq < min_freq)
7962                         ia_freq = 800;
7963                 else
7964                         ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
7965                 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
7966
7967                 I915_WRITE(GEN6_PCODE_DATA,
7968                            (ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) |
7969                            gpu_freq);
7970                 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
7971                            GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
7972                 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
7973                               GEN6_PCODE_READY) == 0, 10)) {
7974                         DRM_ERROR("pcode write of freq table timed out\n");
7975                         continue;
7976                 }
7977         }
7978
7979         mutex_unlock(&dev_priv->dev->struct_mutex);
7980 }
7981
7982 static void ironlake_init_clock_gating(struct drm_device *dev)
7983 {
7984         struct drm_i915_private *dev_priv = dev->dev_private;
7985         uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
7986
7987         /* Required for FBC */
7988         dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE |
7989                 DPFCRUNIT_CLOCK_GATE_DISABLE |
7990                 DPFDUNIT_CLOCK_GATE_DISABLE;
7991         /* Required for CxSR */
7992         dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
7993
7994         I915_WRITE(PCH_3DCGDIS0,
7995                    MARIUNIT_CLOCK_GATE_DISABLE |
7996                    SVSMUNIT_CLOCK_GATE_DISABLE);
7997         I915_WRITE(PCH_3DCGDIS1,
7998                    VFMUNIT_CLOCK_GATE_DISABLE);
7999
8000         I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
8001
8002         /*
8003          * According to the spec the following bits should be set in
8004          * order to enable memory self-refresh
8005          * The bit 22/21 of 0x42004
8006          * The bit 5 of 0x42020
8007          * The bit 15 of 0x45000
8008          */
8009         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8010                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
8011                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8012         I915_WRITE(ILK_DSPCLK_GATE,
8013                    (I915_READ(ILK_DSPCLK_GATE) |
8014                     ILK_DPARB_CLK_GATE));
8015         I915_WRITE(DISP_ARB_CTL,
8016                    (I915_READ(DISP_ARB_CTL) |
8017                     DISP_FBC_WM_DIS));
8018         I915_WRITE(WM3_LP_ILK, 0);
8019         I915_WRITE(WM2_LP_ILK, 0);
8020         I915_WRITE(WM1_LP_ILK, 0);
8021
8022         /*
8023          * Based on the document from hardware guys the following bits
8024          * should be set unconditionally in order to enable FBC.
8025          * The bit 22 of 0x42000
8026          * The bit 22 of 0x42004
8027          * The bit 7,8,9 of 0x42020.
8028          */
8029         if (IS_IRONLAKE_M(dev)) {
8030                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8031                            I915_READ(ILK_DISPLAY_CHICKEN1) |
8032                            ILK_FBCQ_DIS);
8033                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8034                            I915_READ(ILK_DISPLAY_CHICKEN2) |
8035                            ILK_DPARB_GATE);
8036                 I915_WRITE(ILK_DSPCLK_GATE,
8037                            I915_READ(ILK_DSPCLK_GATE) |
8038                            ILK_DPFC_DIS1 |
8039                            ILK_DPFC_DIS2 |
8040                            ILK_CLK_FBC);
8041         }
8042
8043         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8044                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8045                    ILK_ELPIN_409_SELECT);
8046         I915_WRITE(_3D_CHICKEN2,
8047                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8048                    _3D_CHICKEN2_WM_READ_PIPELINED);
8049 }
8050
8051 static void gen6_init_clock_gating(struct drm_device *dev)
8052 {
8053         struct drm_i915_private *dev_priv = dev->dev_private;
8054         int pipe;
8055         uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
8056
8057         I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
8058
8059         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8060                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8061                    ILK_ELPIN_409_SELECT);
8062
8063         I915_WRITE(WM3_LP_ILK, 0);
8064         I915_WRITE(WM2_LP_ILK, 0);
8065         I915_WRITE(WM1_LP_ILK, 0);
8066
8067         /*
8068          * According to the spec the following bits should be
8069          * set in order to enable memory self-refresh and fbc:
8070          * The bit21 and bit22 of 0x42000
8071          * The bit21 and bit22 of 0x42004
8072          * The bit5 and bit7 of 0x42020
8073          * The bit14 of 0x70180
8074          * The bit14 of 0x71180
8075          */
8076         I915_WRITE(ILK_DISPLAY_CHICKEN1,
8077                    I915_READ(ILK_DISPLAY_CHICKEN1) |
8078                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
8079         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8080                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8081                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
8082         I915_WRITE(ILK_DSPCLK_GATE,
8083                    I915_READ(ILK_DSPCLK_GATE) |
8084                    ILK_DPARB_CLK_GATE  |
8085                    ILK_DPFD_CLK_GATE);
8086
8087         for_each_pipe(pipe) {
8088                 I915_WRITE(DSPCNTR(pipe),
8089                            I915_READ(DSPCNTR(pipe)) |
8090                            DISPPLANE_TRICKLE_FEED_DISABLE);
8091                 intel_flush_display_plane(dev_priv, pipe);
8092         }
8093 }
8094
8095 static void ivybridge_init_clock_gating(struct drm_device *dev)
8096 {
8097         struct drm_i915_private *dev_priv = dev->dev_private;
8098         int pipe;
8099         uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
8100
8101         I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
8102
8103         I915_WRITE(WM3_LP_ILK, 0);
8104         I915_WRITE(WM2_LP_ILK, 0);
8105         I915_WRITE(WM1_LP_ILK, 0);
8106
8107         I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
8108
8109         for_each_pipe(pipe) {
8110                 I915_WRITE(DSPCNTR(pipe),
8111                            I915_READ(DSPCNTR(pipe)) |
8112                            DISPPLANE_TRICKLE_FEED_DISABLE);
8113                 intel_flush_display_plane(dev_priv, pipe);
8114         }
8115 }
8116
8117 static void g4x_init_clock_gating(struct drm_device *dev)
8118 {
8119         struct drm_i915_private *dev_priv = dev->dev_private;
8120         uint32_t dspclk_gate;
8121
8122         I915_WRITE(RENCLK_GATE_D1, 0);
8123         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
8124                    GS_UNIT_CLOCK_GATE_DISABLE |
8125                    CL_UNIT_CLOCK_GATE_DISABLE);
8126         I915_WRITE(RAMCLK_GATE_D, 0);
8127         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
8128                 OVRUNIT_CLOCK_GATE_DISABLE |
8129                 OVCUNIT_CLOCK_GATE_DISABLE;
8130         if (IS_GM45(dev))
8131                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
8132         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
8133 }
8134
8135 static void crestline_init_clock_gating(struct drm_device *dev)
8136 {
8137         struct drm_i915_private *dev_priv = dev->dev_private;
8138
8139         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
8140         I915_WRITE(RENCLK_GATE_D2, 0);
8141         I915_WRITE(DSPCLK_GATE_D, 0);
8142         I915_WRITE(RAMCLK_GATE_D, 0);
8143         I915_WRITE16(DEUC, 0);
8144 }
8145
8146 static void broadwater_init_clock_gating(struct drm_device *dev)
8147 {
8148         struct drm_i915_private *dev_priv = dev->dev_private;
8149
8150         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
8151                    I965_RCC_CLOCK_GATE_DISABLE |
8152                    I965_RCPB_CLOCK_GATE_DISABLE |
8153                    I965_ISC_CLOCK_GATE_DISABLE |
8154                    I965_FBC_CLOCK_GATE_DISABLE);
8155         I915_WRITE(RENCLK_GATE_D2, 0);
8156 }
8157
8158 static void gen3_init_clock_gating(struct drm_device *dev)
8159 {
8160         struct drm_i915_private *dev_priv = dev->dev_private;
8161         u32 dstate = I915_READ(D_STATE);
8162
8163         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
8164                 DSTATE_DOT_CLOCK_GATING;
8165         I915_WRITE(D_STATE, dstate);
8166 }
8167
8168 static void i85x_init_clock_gating(struct drm_device *dev)
8169 {
8170         struct drm_i915_private *dev_priv = dev->dev_private;
8171
8172         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
8173 }
8174
8175 static void i830_init_clock_gating(struct drm_device *dev)
8176 {
8177         struct drm_i915_private *dev_priv = dev->dev_private;
8178
8179         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
8180 }
8181
8182 static void ibx_init_clock_gating(struct drm_device *dev)
8183 {
8184         struct drm_i915_private *dev_priv = dev->dev_private;
8185
8186         /*
8187          * On Ibex Peak and Cougar Point, we need to disable clock
8188          * gating for the panel power sequencer or it will fail to
8189          * start up when no ports are active.
8190          */
8191         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8192 }
8193
8194 static void cpt_init_clock_gating(struct drm_device *dev)
8195 {
8196         struct drm_i915_private *dev_priv = dev->dev_private;
8197         int pipe;
8198
8199         /*
8200          * On Ibex Peak and Cougar Point, we need to disable clock
8201          * gating for the panel power sequencer or it will fail to
8202          * start up when no ports are active.
8203          */
8204         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8205         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
8206                    DPLS_EDP_PPS_FIX_DIS);
8207         /* Without this, mode sets may fail silently on FDI */
8208         for_each_pipe(pipe)
8209                 I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_AUTOTRAIN_GEN_STALL_DIS);
8210 }
8211
8212 static void ironlake_teardown_rc6(struct drm_device *dev)
8213 {
8214         struct drm_i915_private *dev_priv = dev->dev_private;
8215
8216         if (dev_priv->renderctx) {
8217                 i915_gem_object_unpin(dev_priv->renderctx);
8218                 drm_gem_object_unreference(&dev_priv->renderctx->base);
8219                 dev_priv->renderctx = NULL;
8220         }
8221
8222         if (dev_priv->pwrctx) {
8223                 i915_gem_object_unpin(dev_priv->pwrctx);
8224                 drm_gem_object_unreference(&dev_priv->pwrctx->base);
8225                 dev_priv->pwrctx = NULL;
8226         }
8227 }
8228
8229 static void ironlake_disable_rc6(struct drm_device *dev)
8230 {
8231         struct drm_i915_private *dev_priv = dev->dev_private;
8232
8233         if (I915_READ(PWRCTXA)) {
8234                 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
8235                 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
8236                 wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
8237                          50);
8238
8239                 I915_WRITE(PWRCTXA, 0);
8240                 POSTING_READ(PWRCTXA);
8241
8242                 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
8243                 POSTING_READ(RSTDBYCTL);
8244         }
8245
8246         ironlake_teardown_rc6(dev);
8247 }
8248
8249 static int ironlake_setup_rc6(struct drm_device *dev)
8250 {
8251         struct drm_i915_private *dev_priv = dev->dev_private;
8252
8253         if (dev_priv->renderctx == NULL)
8254                 dev_priv->renderctx = intel_alloc_context_page(dev);
8255         if (!dev_priv->renderctx)
8256                 return -ENOMEM;
8257
8258         if (dev_priv->pwrctx == NULL)
8259                 dev_priv->pwrctx = intel_alloc_context_page(dev);
8260         if (!dev_priv->pwrctx) {
8261                 ironlake_teardown_rc6(dev);
8262                 return -ENOMEM;
8263         }
8264
8265         return 0;
8266 }
8267
8268 void ironlake_enable_rc6(struct drm_device *dev)
8269 {
8270         struct drm_i915_private *dev_priv = dev->dev_private;
8271         int ret;
8272
8273         /* rc6 disabled by default due to repeated reports of hanging during
8274          * boot and resume.
8275          */
8276         if (!i915_enable_rc6)
8277                 return;
8278
8279         mutex_lock(&dev->struct_mutex);
8280         ret = ironlake_setup_rc6(dev);
8281         if (ret) {
8282                 mutex_unlock(&dev->struct_mutex);
8283                 return;
8284         }
8285
8286         /*
8287          * GPU can automatically power down the render unit if given a page
8288          * to save state.
8289          */
8290         ret = BEGIN_LP_RING(6);
8291         if (ret) {
8292                 ironlake_teardown_rc6(dev);
8293                 mutex_unlock(&dev->struct_mutex);
8294                 return;
8295         }
8296
8297         OUT_RING(MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
8298         OUT_RING(MI_SET_CONTEXT);
8299         OUT_RING(dev_priv->renderctx->gtt_offset |
8300                  MI_MM_SPACE_GTT |
8301                  MI_SAVE_EXT_STATE_EN |
8302                  MI_RESTORE_EXT_STATE_EN |
8303                  MI_RESTORE_INHIBIT);
8304         OUT_RING(MI_SUSPEND_FLUSH);
8305         OUT_RING(MI_NOOP);
8306         OUT_RING(MI_FLUSH);
8307         ADVANCE_LP_RING();
8308
8309         /*
8310          * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
8311          * does an implicit flush, combined with MI_FLUSH above, it should be
8312          * safe to assume that renderctx is valid
8313          */
8314         ret = intel_wait_ring_idle(LP_RING(dev_priv));
8315         if (ret) {
8316                 DRM_ERROR("failed to enable ironlake power power savings\n");
8317                 ironlake_teardown_rc6(dev);
8318                 mutex_unlock(&dev->struct_mutex);
8319                 return;
8320         }
8321
8322         I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset | PWRCTX_EN);
8323         I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
8324         mutex_unlock(&dev->struct_mutex);
8325 }
8326
8327 void intel_init_clock_gating(struct drm_device *dev)
8328 {
8329         struct drm_i915_private *dev_priv = dev->dev_private;
8330
8331         dev_priv->display.init_clock_gating(dev);
8332
8333         if (dev_priv->display.init_pch_clock_gating)
8334                 dev_priv->display.init_pch_clock_gating(dev);
8335 }
8336
8337 /* Set up chip specific display functions */
8338 static void intel_init_display(struct drm_device *dev)
8339 {
8340         struct drm_i915_private *dev_priv = dev->dev_private;
8341
8342         /* We always want a DPMS function */
8343         if (HAS_PCH_SPLIT(dev)) {
8344                 dev_priv->display.dpms = ironlake_crtc_dpms;
8345                 dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
8346                 dev_priv->display.update_plane = ironlake_update_plane;
8347         } else {
8348                 dev_priv->display.dpms = i9xx_crtc_dpms;
8349                 dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
8350                 dev_priv->display.update_plane = i9xx_update_plane;
8351         }
8352
8353         if (I915_HAS_FBC(dev)) {
8354                 if (HAS_PCH_SPLIT(dev)) {
8355                         dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
8356                         dev_priv->display.enable_fbc = ironlake_enable_fbc;
8357                         dev_priv->display.disable_fbc = ironlake_disable_fbc;
8358                 } else if (IS_GM45(dev)) {
8359                         dev_priv->display.fbc_enabled = g4x_fbc_enabled;
8360                         dev_priv->display.enable_fbc = g4x_enable_fbc;
8361                         dev_priv->display.disable_fbc = g4x_disable_fbc;
8362                 } else if (IS_CRESTLINE(dev)) {
8363                         dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
8364                         dev_priv->display.enable_fbc = i8xx_enable_fbc;
8365                         dev_priv->display.disable_fbc = i8xx_disable_fbc;
8366                 }
8367                 /* 855GM needs testing */
8368         }
8369
8370         /* Returns the core display clock speed */
8371         if (IS_I945G(dev) || (IS_G33(dev) && !IS_PINEVIEW_M(dev)))
8372                 dev_priv->display.get_display_clock_speed =
8373                         i945_get_display_clock_speed;
8374         else if (IS_I915G(dev))
8375                 dev_priv->display.get_display_clock_speed =
8376                         i915_get_display_clock_speed;
8377         else if (IS_I945GM(dev) || IS_845G(dev) || IS_PINEVIEW_M(dev))
8378                 dev_priv->display.get_display_clock_speed =
8379                         i9xx_misc_get_display_clock_speed;
8380         else if (IS_I915GM(dev))
8381                 dev_priv->display.get_display_clock_speed =
8382                         i915gm_get_display_clock_speed;
8383         else if (IS_I865G(dev))
8384                 dev_priv->display.get_display_clock_speed =
8385                         i865_get_display_clock_speed;
8386         else if (IS_I85X(dev))
8387                 dev_priv->display.get_display_clock_speed =
8388                         i855_get_display_clock_speed;
8389         else /* 852, 830 */
8390                 dev_priv->display.get_display_clock_speed =
8391                         i830_get_display_clock_speed;
8392
8393         /* For FIFO watermark updates */
8394         if (HAS_PCH_SPLIT(dev)) {
8395                 if (HAS_PCH_IBX(dev))
8396                         dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
8397                 else if (HAS_PCH_CPT(dev))
8398                         dev_priv->display.init_pch_clock_gating = cpt_init_clock_gating;
8399
8400                 if (IS_GEN5(dev)) {
8401                         if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
8402                                 dev_priv->display.update_wm = ironlake_update_wm;
8403                         else {
8404                                 DRM_DEBUG_KMS("Failed to get proper latency. "
8405                                               "Disable CxSR\n");
8406                                 dev_priv->display.update_wm = NULL;
8407                         }
8408                         dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
8409                         dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
8410                         dev_priv->display.write_eld = ironlake_write_eld;
8411                 } else if (IS_GEN6(dev)) {
8412                         if (SNB_READ_WM0_LATENCY()) {
8413                                 dev_priv->display.update_wm = sandybridge_update_wm;
8414                         } else {
8415                                 DRM_DEBUG_KMS("Failed to read display plane latency. "
8416                                               "Disable CxSR\n");
8417                                 dev_priv->display.update_wm = NULL;
8418                         }
8419                         dev_priv->display.fdi_link_train = gen6_fdi_link_train;
8420                         dev_priv->display.init_clock_gating = gen6_init_clock_gating;
8421                         dev_priv->display.write_eld = ironlake_write_eld;
8422                 } else if (IS_IVYBRIDGE(dev)) {
8423                         /* FIXME: detect B0+ stepping and use auto training */
8424                         dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
8425                         if (SNB_READ_WM0_LATENCY()) {
8426                                 dev_priv->display.update_wm = sandybridge_update_wm;
8427                         } else {
8428                                 DRM_DEBUG_KMS("Failed to read display plane latency. "
8429                                               "Disable CxSR\n");
8430                                 dev_priv->display.update_wm = NULL;
8431                         }
8432                         dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
8433                         dev_priv->display.write_eld = ironlake_write_eld;
8434                 } else
8435                         dev_priv->display.update_wm = NULL;
8436         } else if (IS_PINEVIEW(dev)) {
8437                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
8438                                             dev_priv->is_ddr3,
8439                                             dev_priv->fsb_freq,
8440                                             dev_priv->mem_freq)) {
8441                         DRM_INFO("failed to find known CxSR latency "
8442                                  "(found ddr%s fsb freq %d, mem freq %d), "
8443                                  "disabling CxSR\n",
8444                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
8445                                  dev_priv->fsb_freq, dev_priv->mem_freq);
8446                         /* Disable CxSR and never update its watermark again */
8447                         pineview_disable_cxsr(dev);
8448                         dev_priv->display.update_wm = NULL;
8449                 } else
8450                         dev_priv->display.update_wm = pineview_update_wm;
8451                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
8452         } else if (IS_G4X(dev)) {
8453                 dev_priv->display.write_eld = g4x_write_eld;
8454                 dev_priv->display.update_wm = g4x_update_wm;
8455                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
8456         } else if (IS_GEN4(dev)) {
8457                 dev_priv->display.update_wm = i965_update_wm;
8458                 if (IS_CRESTLINE(dev))
8459                         dev_priv->display.init_clock_gating = crestline_init_clock_gating;
8460                 else if (IS_BROADWATER(dev))
8461                         dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
8462         } else if (IS_GEN3(dev)) {
8463                 dev_priv->display.update_wm = i9xx_update_wm;
8464                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
8465                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
8466         } else if (IS_I865G(dev)) {
8467                 dev_priv->display.update_wm = i830_update_wm;
8468                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
8469                 dev_priv->display.get_fifo_size = i830_get_fifo_size;
8470         } else if (IS_I85X(dev)) {
8471                 dev_priv->display.update_wm = i9xx_update_wm;
8472                 dev_priv->display.get_fifo_size = i85x_get_fifo_size;
8473                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
8474         } else {
8475                 dev_priv->display.update_wm = i830_update_wm;
8476                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
8477                 if (IS_845G(dev))
8478                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
8479                 else
8480                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
8481         }
8482
8483         /* Default just returns -ENODEV to indicate unsupported */
8484         dev_priv->display.queue_flip = intel_default_queue_flip;
8485
8486         switch (INTEL_INFO(dev)->gen) {
8487         case 2:
8488                 dev_priv->display.queue_flip = intel_gen2_queue_flip;
8489                 break;
8490
8491         case 3:
8492                 dev_priv->display.queue_flip = intel_gen3_queue_flip;
8493                 break;
8494
8495         case 4:
8496         case 5:
8497                 dev_priv->display.queue_flip = intel_gen4_queue_flip;
8498                 break;
8499
8500         case 6:
8501                 dev_priv->display.queue_flip = intel_gen6_queue_flip;
8502                 break;
8503         case 7:
8504                 dev_priv->display.queue_flip = intel_gen7_queue_flip;
8505                 break;
8506         }
8507 }
8508
8509 /*
8510  * Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend,
8511  * resume, or other times.  This quirk makes sure that's the case for
8512  * affected systems.
8513  */
8514 static void quirk_pipea_force(struct drm_device *dev)
8515 {
8516         struct drm_i915_private *dev_priv = dev->dev_private;
8517
8518         dev_priv->quirks |= QUIRK_PIPEA_FORCE;
8519         DRM_DEBUG_DRIVER("applying pipe a force quirk\n");
8520 }
8521
8522 /*
8523  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
8524  */
8525 static void quirk_ssc_force_disable(struct drm_device *dev)
8526 {
8527         struct drm_i915_private *dev_priv = dev->dev_private;
8528         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
8529 }
8530
8531 struct intel_quirk {
8532         int device;
8533         int subsystem_vendor;
8534         int subsystem_device;
8535         void (*hook)(struct drm_device *dev);
8536 };
8537
8538 struct intel_quirk intel_quirks[] = {
8539         /* HP Compaq 2730p needs pipe A force quirk (LP: #291555) */
8540         { 0x2a42, 0x103c, 0x30eb, quirk_pipea_force },
8541         /* HP Mini needs pipe A force quirk (LP: #322104) */
8542         { 0x27ae, 0x103c, 0x361a, quirk_pipea_force },
8543
8544         /* Thinkpad R31 needs pipe A force quirk */
8545         { 0x3577, 0x1014, 0x0505, quirk_pipea_force },
8546         /* Toshiba Protege R-205, S-209 needs pipe A force quirk */
8547         { 0x2592, 0x1179, 0x0001, quirk_pipea_force },
8548
8549         /* ThinkPad X30 needs pipe A force quirk (LP: #304614) */
8550         { 0x3577,  0x1014, 0x0513, quirk_pipea_force },
8551         /* ThinkPad X40 needs pipe A force quirk */
8552
8553         /* ThinkPad T60 needs pipe A force quirk (bug #16494) */
8554         { 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
8555
8556         /* 855 & before need to leave pipe A & dpll A up */
8557         { 0x3582, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
8558         { 0x2562, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
8559
8560         /* Lenovo U160 cannot use SSC on LVDS */
8561         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
8562
8563         /* Sony Vaio Y cannot use SSC on LVDS */
8564         { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
8565 };
8566
8567 static void intel_init_quirks(struct drm_device *dev)
8568 {
8569         struct pci_dev *d = dev->pdev;
8570         int i;
8571
8572         for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
8573                 struct intel_quirk *q = &intel_quirks[i];
8574
8575                 if (d->device == q->device &&
8576                     (d->subsystem_vendor == q->subsystem_vendor ||
8577                      q->subsystem_vendor == PCI_ANY_ID) &&
8578                     (d->subsystem_device == q->subsystem_device ||
8579                      q->subsystem_device == PCI_ANY_ID))
8580                         q->hook(dev);
8581         }
8582 }
8583
8584 /* Disable the VGA plane that we never use */
8585 static void i915_disable_vga(struct drm_device *dev)
8586 {
8587         struct drm_i915_private *dev_priv = dev->dev_private;
8588         u8 sr1;
8589         u32 vga_reg;
8590
8591         if (HAS_PCH_SPLIT(dev))
8592                 vga_reg = CPU_VGACNTRL;
8593         else
8594                 vga_reg = VGACNTRL;
8595
8596         vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
8597         outb(1, VGA_SR_INDEX);
8598         sr1 = inb(VGA_SR_DATA);
8599         outb(sr1 | 1<<5, VGA_SR_DATA);
8600         vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
8601         udelay(300);
8602
8603         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
8604         POSTING_READ(vga_reg);
8605 }
8606
8607 void intel_modeset_init(struct drm_device *dev)
8608 {
8609         struct drm_i915_private *dev_priv = dev->dev_private;
8610         int i;
8611
8612         drm_mode_config_init(dev);
8613
8614         dev->mode_config.min_width = 0;
8615         dev->mode_config.min_height = 0;
8616
8617         dev->mode_config.funcs = (void *)&intel_mode_funcs;
8618
8619         intel_init_quirks(dev);
8620
8621         intel_init_display(dev);
8622
8623         if (IS_GEN2(dev)) {
8624                 dev->mode_config.max_width = 2048;
8625                 dev->mode_config.max_height = 2048;
8626         } else if (IS_GEN3(dev)) {
8627                 dev->mode_config.max_width = 4096;
8628                 dev->mode_config.max_height = 4096;
8629         } else {
8630                 dev->mode_config.max_width = 8192;
8631                 dev->mode_config.max_height = 8192;
8632         }
8633         dev->mode_config.fb_base = dev->agp->base;
8634
8635         DRM_DEBUG_KMS("%d display pipe%s available.\n",
8636                       dev_priv->num_pipe, dev_priv->num_pipe > 1 ? "s" : "");
8637
8638         for (i = 0; i < dev_priv->num_pipe; i++) {
8639                 intel_crtc_init(dev, i);
8640         }
8641
8642         /* Just disable it once at startup */
8643         i915_disable_vga(dev);
8644         intel_setup_outputs(dev);
8645
8646         intel_init_clock_gating(dev);
8647
8648         if (IS_IRONLAKE_M(dev)) {
8649                 ironlake_enable_drps(dev);
8650                 intel_init_emon(dev);
8651         }
8652
8653         if (IS_GEN6(dev) || IS_GEN7(dev)) {
8654                 gen6_enable_rps(dev_priv);
8655                 gen6_update_ring_freq(dev_priv);
8656         }
8657
8658         INIT_WORK(&dev_priv->idle_work, intel_idle_update);
8659         setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer,
8660                     (unsigned long)dev);
8661 }
8662
8663 void intel_modeset_gem_init(struct drm_device *dev)
8664 {
8665         if (IS_IRONLAKE_M(dev))
8666                 ironlake_enable_rc6(dev);
8667
8668         intel_setup_overlay(dev);
8669 }
8670
8671 void intel_modeset_cleanup(struct drm_device *dev)
8672 {
8673         struct drm_i915_private *dev_priv = dev->dev_private;
8674         struct drm_crtc *crtc;
8675         struct intel_crtc *intel_crtc;
8676
8677         drm_kms_helper_poll_fini(dev);
8678         mutex_lock(&dev->struct_mutex);
8679
8680         intel_unregister_dsm_handler();
8681
8682
8683         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
8684                 /* Skip inactive CRTCs */
8685                 if (!crtc->fb)
8686                         continue;
8687
8688                 intel_crtc = to_intel_crtc(crtc);
8689                 intel_increase_pllclock(crtc);
8690         }
8691
8692         intel_disable_fbc(dev);
8693
8694         if (IS_IRONLAKE_M(dev))
8695                 ironlake_disable_drps(dev);
8696         if (IS_GEN6(dev) || IS_GEN7(dev))
8697                 gen6_disable_rps(dev);
8698
8699         if (IS_IRONLAKE_M(dev))
8700                 ironlake_disable_rc6(dev);
8701
8702         mutex_unlock(&dev->struct_mutex);
8703
8704         /* Disable the irq before mode object teardown, for the irq might
8705          * enqueue unpin/hotplug work. */
8706         drm_irq_uninstall(dev);
8707         cancel_work_sync(&dev_priv->hotplug_work);
8708         cancel_work_sync(&dev_priv->rps_work);
8709
8710         /* flush any delayed tasks or pending work */
8711         flush_scheduled_work();
8712
8713         /* Shut off idle work before the crtcs get freed. */
8714         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
8715                 intel_crtc = to_intel_crtc(crtc);
8716                 del_timer_sync(&intel_crtc->idle_timer);
8717         }
8718         del_timer_sync(&dev_priv->idle_timer);
8719         cancel_work_sync(&dev_priv->idle_work);
8720
8721         drm_mode_config_cleanup(dev);
8722 }
8723
8724 /*
8725  * Return which encoder is currently attached for connector.
8726  */
8727 struct drm_encoder *intel_best_encoder(struct drm_connector *connector)
8728 {
8729         return &intel_attached_encoder(connector)->base;
8730 }
8731
8732 void intel_connector_attach_encoder(struct intel_connector *connector,
8733                                     struct intel_encoder *encoder)
8734 {
8735         connector->encoder = encoder;
8736         drm_mode_connector_attach_encoder(&connector->base,
8737                                           &encoder->base);
8738 }
8739
8740 /*
8741  * set vga decode state - true == enable VGA decode
8742  */
8743 int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
8744 {
8745         struct drm_i915_private *dev_priv = dev->dev_private;
8746         u16 gmch_ctrl;
8747
8748         pci_read_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, &gmch_ctrl);
8749         if (state)
8750                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
8751         else
8752                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
8753         pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl);
8754         return 0;
8755 }
8756
8757 #ifdef CONFIG_DEBUG_FS
8758 #include <linux/seq_file.h>
8759
8760 struct intel_display_error_state {
8761         struct intel_cursor_error_state {
8762                 u32 control;
8763                 u32 position;
8764                 u32 base;
8765                 u32 size;
8766         } cursor[2];
8767
8768         struct intel_pipe_error_state {
8769                 u32 conf;
8770                 u32 source;
8771
8772                 u32 htotal;
8773                 u32 hblank;
8774                 u32 hsync;
8775                 u32 vtotal;
8776                 u32 vblank;
8777                 u32 vsync;
8778         } pipe[2];
8779
8780         struct intel_plane_error_state {
8781                 u32 control;
8782                 u32 stride;
8783                 u32 size;
8784                 u32 pos;
8785                 u32 addr;
8786                 u32 surface;
8787                 u32 tile_offset;
8788         } plane[2];
8789 };
8790
8791 struct intel_display_error_state *
8792 intel_display_capture_error_state(struct drm_device *dev)
8793 {
8794         drm_i915_private_t *dev_priv = dev->dev_private;
8795         struct intel_display_error_state *error;
8796         int i;
8797
8798         error = kmalloc(sizeof(*error), GFP_ATOMIC);
8799         if (error == NULL)
8800                 return NULL;
8801
8802         for (i = 0; i < 2; i++) {
8803                 error->cursor[i].control = I915_READ(CURCNTR(i));
8804                 error->cursor[i].position = I915_READ(CURPOS(i));
8805                 error->cursor[i].base = I915_READ(CURBASE(i));
8806
8807                 error->plane[i].control = I915_READ(DSPCNTR(i));
8808                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
8809                 error->plane[i].size = I915_READ(DSPSIZE(i));
8810                 error->plane[i].pos = I915_READ(DSPPOS(i));
8811                 error->plane[i].addr = I915_READ(DSPADDR(i));
8812                 if (INTEL_INFO(dev)->gen >= 4) {
8813                         error->plane[i].surface = I915_READ(DSPSURF(i));
8814                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
8815                 }
8816
8817                 error->pipe[i].conf = I915_READ(PIPECONF(i));
8818                 error->pipe[i].source = I915_READ(PIPESRC(i));
8819                 error->pipe[i].htotal = I915_READ(HTOTAL(i));
8820                 error->pipe[i].hblank = I915_READ(HBLANK(i));
8821                 error->pipe[i].hsync = I915_READ(HSYNC(i));
8822                 error->pipe[i].vtotal = I915_READ(VTOTAL(i));
8823                 error->pipe[i].vblank = I915_READ(VBLANK(i));
8824                 error->pipe[i].vsync = I915_READ(VSYNC(i));
8825         }
8826
8827         return error;
8828 }
8829
8830 void
8831 intel_display_print_error_state(struct seq_file *m,
8832                                 struct drm_device *dev,
8833                                 struct intel_display_error_state *error)
8834 {
8835         int i;
8836
8837         for (i = 0; i < 2; i++) {
8838                 seq_printf(m, "Pipe [%d]:\n", i);
8839                 seq_printf(m, "  CONF: %08x\n", error->pipe[i].conf);
8840                 seq_printf(m, "  SRC: %08x\n", error->pipe[i].source);
8841                 seq_printf(m, "  HTOTAL: %08x\n", error->pipe[i].htotal);
8842                 seq_printf(m, "  HBLANK: %08x\n", error->pipe[i].hblank);
8843                 seq_printf(m, "  HSYNC: %08x\n", error->pipe[i].hsync);
8844                 seq_printf(m, "  VTOTAL: %08x\n", error->pipe[i].vtotal);
8845                 seq_printf(m, "  VBLANK: %08x\n", error->pipe[i].vblank);
8846                 seq_printf(m, "  VSYNC: %08x\n", error->pipe[i].vsync);
8847
8848                 seq_printf(m, "Plane [%d]:\n", i);
8849                 seq_printf(m, "  CNTR: %08x\n", error->plane[i].control);
8850                 seq_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
8851                 seq_printf(m, "  SIZE: %08x\n", error->plane[i].size);
8852                 seq_printf(m, "  POS: %08x\n", error->plane[i].pos);
8853                 seq_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
8854                 if (INTEL_INFO(dev)->gen >= 4) {
8855                         seq_printf(m, "  SURF: %08x\n", error->plane[i].surface);
8856                         seq_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
8857                 }
8858
8859                 seq_printf(m, "Cursor [%d]:\n", i);
8860                 seq_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
8861                 seq_printf(m, "  POS: %08x\n", error->cursor[i].position);
8862                 seq_printf(m, "  BASE: %08x\n", error->cursor[i].base);
8863         }
8864 }
8865 #endif