drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 static u32 i915_gem_get_seqno(struct drm_device *dev)
  38 {
  39         drm_i915_private_t *dev_priv = dev->dev_private;
  40         u32 seqno;
  41
  42         seqno = dev_priv->next_seqno;
  43
  44         /* reserve 0 for non-seqno */
  45         if (++dev_priv->next_seqno == 0)
  46                 dev_priv->next_seqno = 1;
  47
  48         return seqno;
  49 }
  50
  51 static void
  52 render_ring_flush(struct intel_ring_buffer *ring,
  53                   u32   invalidate_domains,
  54                   u32   flush_domains)
  55 {
  56         struct drm_device *dev = ring->dev;
  57         drm_i915_private_t *dev_priv = dev->dev_private;
  58         u32 cmd;
  59
  60 #if WATCH_EXEC
  61         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
  62                   invalidate_domains, flush_domains);
  63 #endif
  64
  65         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
  66                                      invalidate_domains, flush_domains);
  67
  68         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
  69                 /*
  70                  * read/write caches:
  71                  *
  72                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  73                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  74                  * also flushed at 2d versus 3d pipeline switches.
  75                  *
  76                  * read-only caches:
  77                  *
  78                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  79                  * MI_READ_FLUSH is set, and is always flushed on 965.
  80                  *
  81                  * I915_GEM_DOMAIN_COMMAND may not exist?
  82                  *
  83                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  84                  * invalidated when MI_EXE_FLUSH is set.
  85                  *
  86                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  87                  * invalidated with every MI_FLUSH.
  88                  *
  89                  * TLBs:
  90                  *
  91                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  92                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  93                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  94                  * are flushed at any MI_FLUSH.
  95                  */
  96
  97                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  98                 if ((invalidate_domains|flush_domains) &
  99                     I915_GEM_DOMAIN_RENDER)
 100                         cmd &= ~MI_NO_WRITE_FLUSH;
 101                 if (INTEL_INFO(dev)->gen < 4) {
 102                         /*
 103                          * On the 965, the sampler cache always gets flushed
 104                          * and this bit is reserved.
 105                          */
 106                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 107                                 cmd |= MI_READ_FLUSH;
 108                 }
 109                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 110                         cmd |= MI_EXE_FLUSH;
 111
 112 #if WATCH_EXEC
 113                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 114 #endif
 115                 if (intel_ring_begin(ring, 2) == 0) {
 116                         intel_ring_emit(ring, cmd);
 117                         intel_ring_emit(ring, MI_NOOP);
 118                         intel_ring_advance(ring);
 119                 }
 120         }
 121 }
 122
 123 static void ring_write_tail(struct intel_ring_buffer *ring,
 124                             u32 value)
 125 {
 126         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 127         I915_WRITE_TAIL(ring, value);
 128 }
 129
 130 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 131 {
 132         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 133         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 134                         RING_ACTHD(ring->mmio_base) : ACTHD;
 135
 136         return I915_READ(acthd_reg);
 137 }
 138
 139 static int init_ring_common(struct intel_ring_buffer *ring)
 140 {
 141         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 142         struct drm_i915_gem_object *obj_priv = to_intel_bo(ring->gem_object);
 143         u32 head;
 144
 145         /* Stop the ring if it's running. */
 146         I915_WRITE_CTL(ring, 0);
 147         I915_WRITE_HEAD(ring, 0);
 148         ring->write_tail(ring, 0);
 149
 150         /* Initialize the ring. */
 151         I915_WRITE_START(ring, obj_priv->gtt_offset);
 152         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 153
 154         /* G45 ring initialization fails to reset head to zero */
 155         if (head != 0) {
 156                 DRM_ERROR("%s head not reset to zero "
 157                                 "ctl %08x head %08x tail %08x start %08x\n",
 158                                 ring->name,
 159                                 I915_READ_CTL(ring),
 160                                 I915_READ_HEAD(ring),
 161                                 I915_READ_TAIL(ring),
 162                                 I915_READ_START(ring));
 163
 164                 I915_WRITE_HEAD(ring, 0);
 165
 166                 DRM_ERROR("%s head forced to zero "
 167                                 "ctl %08x head %08x tail %08x start %08x\n",
 168                                 ring->name,
 169                                 I915_READ_CTL(ring),
 170                                 I915_READ_HEAD(ring),
 171                                 I915_READ_TAIL(ring),
 172                                 I915_READ_START(ring));
 173         }
 174
 175         I915_WRITE_CTL(ring,
 176                         ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
 177                         | RING_NO_REPORT | RING_VALID);
 178
 179         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 180         /* If the head is still not zero, the ring is dead */
 181         if (head != 0) {
 182                 DRM_ERROR("%s initialization failed "
 183                                 "ctl %08x head %08x tail %08x start %08x\n",
 184                                 ring->name,
 185                                 I915_READ_CTL(ring),
 186                                 I915_READ_HEAD(ring),
 187                                 I915_READ_TAIL(ring),
 188                                 I915_READ_START(ring));
 189                 return -EIO;
 190         }
 191
 192         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 193                 i915_kernel_lost_context(ring->dev);
 194         else {
 195                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 196                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 197                 ring->space = ring->head - (ring->tail + 8);
 198                 if (ring->space < 0)
 199                         ring->space += ring->size;
 200         }
 201         return 0;
 202 }
 203
 204 static int init_render_ring(struct intel_ring_buffer *ring)
 205 {
 206         struct drm_device *dev = ring->dev;
 207         int ret = init_ring_common(ring);
 208
 209         if (INTEL_INFO(dev)->gen > 3) {
 210                 drm_i915_private_t *dev_priv = dev->dev_private;
 211                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
 212                 if (IS_GEN6(dev))
 213                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
 214                 I915_WRITE(MI_MODE, mode);
 215         }
 216
 217         return ret;
 218 }
 219
 220 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 221 do {                                                                    \
 222         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
 223                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
 224         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 225         intel_ring_emit(ring__, 0);                                                     \
 226         intel_ring_emit(ring__, 0);                                                     \
 227 } while (0)
 228
 229 /**
 230  * Creates a new sequence number, emitting a write of it to the status page
 231  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 232  *
 233  * Must be called with struct_lock held.
 234  *
 235  * Returned sequence numbers are nonzero on success.
 236  */
 237 static u32
 238 render_ring_add_request(struct intel_ring_buffer *ring,
 239                         u32 flush_domains)
 240 {
 241         struct drm_device *dev = ring->dev;
 242         drm_i915_private_t *dev_priv = dev->dev_private;
 243         u32 seqno;
 244
 245         seqno = i915_gem_get_seqno(dev);
 246
 247         if (IS_GEN6(dev)) {
 248                 if (intel_ring_begin(ring, 6) == 0) {
 249                         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | 3);
 250                         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE |
 251                                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
 252                                         PIPE_CONTROL_NOTIFY);
 253                         intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 254                         intel_ring_emit(ring, seqno);
 255                         intel_ring_emit(ring, 0);
 256                         intel_ring_emit(ring, 0);
 257                         intel_ring_advance(ring);
 258                 }
 259         } else if (HAS_PIPE_CONTROL(dev)) {
 260                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
 261
 262                 /*
 263                  * Workaround qword write incoherence by flushing the
 264                  * PIPE_NOTIFY buffers out to memory before requesting
 265                  * an interrupt.
 266                  */
 267                 if (intel_ring_begin(ring, 32) == 0) {
 268                         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 269                                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
 270                         intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 271                         intel_ring_emit(ring, seqno);
 272                         intel_ring_emit(ring, 0);
 273                         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 274                         scratch_addr += 128; /* write to separate cachelines */
 275                         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 276                         scratch_addr += 128;
 277                         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 278                         scratch_addr += 128;
 279                         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 280                         scratch_addr += 128;
 281                         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 282                         scratch_addr += 128;
 283                         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 284                         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 285                                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
 286                                         PIPE_CONTROL_NOTIFY);
 287                         intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 288                         intel_ring_emit(ring, seqno);
 289                         intel_ring_emit(ring, 0);
 290                         intel_ring_advance(ring);
 291                 }
 292         } else {
 293                 if (intel_ring_begin(ring, 4) == 0) {
 294                         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 295                         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 296                         intel_ring_emit(ring, seqno);
 297
 298                         intel_ring_emit(ring, MI_USER_INTERRUPT);
 299                         intel_ring_advance(ring);
 300                 }
 301         }
 302         return seqno;
 303 }
 304
 305 static u32
 306 render_ring_get_seqno(struct intel_ring_buffer *ring)
 307 {
 308         struct drm_device *dev = ring->dev;
 309         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 310         if (HAS_PIPE_CONTROL(dev))
 311                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
 312         else
 313                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 314 }
 315
 316 static void
 317 render_ring_get_user_irq(struct intel_ring_buffer *ring)
 318 {
 319         struct drm_device *dev = ring->dev;
 320         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 321         unsigned long irqflags;
 322
 323         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 324         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
 325                 if (HAS_PCH_SPLIT(dev))
 326                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 327                 else
 328                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 329         }
 330         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 331 }
 332
 333 static void
 334 render_ring_put_user_irq(struct intel_ring_buffer *ring)
 335 {
 336         struct drm_device *dev = ring->dev;
 337         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 338         unsigned long irqflags;
 339
 340         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 341         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
 342         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
 343                 if (HAS_PCH_SPLIT(dev))
 344                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 345                 else
 346                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 347         }
 348         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 349 }
 350
 351 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 352 {
 353         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 354         u32 mmio = IS_GEN6(ring->dev) ?
 355                 RING_HWS_PGA_GEN6(ring->mmio_base) :
 356                 RING_HWS_PGA(ring->mmio_base);
 357         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 358         POSTING_READ(mmio);
 359 }
 360
 361 static void
 362 bsd_ring_flush(struct intel_ring_buffer *ring,
 363                u32     invalidate_domains,
 364                u32     flush_domains)
 365 {
 366         if (intel_ring_begin(ring, 2) == 0) {
 367                 intel_ring_emit(ring, MI_FLUSH);
 368                 intel_ring_emit(ring, MI_NOOP);
 369                 intel_ring_advance(ring);
 370         }
 371 }
 372
 373 static u32
 374 ring_add_request(struct intel_ring_buffer *ring,
 375                  u32 flush_domains)
 376 {
 377         u32 seqno;
 378
 379         seqno = i915_gem_get_seqno(ring->dev);
 380
 381         if (intel_ring_begin(ring, 4) == 0) {
 382                 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 383                 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 384                 intel_ring_emit(ring, seqno);
 385                 intel_ring_emit(ring, MI_USER_INTERRUPT);
 386                 intel_ring_advance(ring);
 387         }
 388
 389         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
 390
 391         return seqno;
 392 }
 393
 394 static void
 395 bsd_ring_get_user_irq(struct intel_ring_buffer *ring)
 396 {
 397         /* do nothing */
 398 }
 399 static void
 400 bsd_ring_put_user_irq(struct intel_ring_buffer *ring)
 401 {
 402         /* do nothing */
 403 }
 404
 405 static u32
 406 ring_status_page_get_seqno(struct intel_ring_buffer *ring)
 407 {
 408         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 409 }
 410
 411 static int
 412 ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 413                          struct drm_i915_gem_execbuffer2 *exec,
 414                          struct drm_clip_rect *cliprects,
 415                          uint64_t exec_offset)
 416 {
 417         uint32_t exec_start;
 418         int ret;
 419
 420         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 421
 422         ret = intel_ring_begin(ring, 2);
 423         if (ret)
 424                 return ret;
 425
 426         intel_ring_emit(ring,
 427                         MI_BATCH_BUFFER_START |
 428                         (2 << 6) |
 429                         MI_BATCH_NON_SECURE_I965);
 430         intel_ring_emit(ring, exec_start);
 431         intel_ring_advance(ring);
 432
 433         return 0;
 434 }
 435
 436 static int
 437 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 438                                 struct drm_i915_gem_execbuffer2 *exec,
 439                                 struct drm_clip_rect *cliprects,
 440                                 uint64_t exec_offset)
 441 {
 442         struct drm_device *dev = ring->dev;
 443         drm_i915_private_t *dev_priv = dev->dev_private;
 444         int nbox = exec->num_cliprects;
 445         uint32_t exec_start, exec_len;
 446         int i, count, ret;
 447
 448         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 449         exec_len = (uint32_t) exec->batch_len;
 450
 451         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
 452
 453         count = nbox ? nbox : 1;
 454         for (i = 0; i < count; i++) {
 455                 if (i < nbox) {
 456                         ret = i915_emit_box(dev, cliprects, i,
 457                                             exec->DR1, exec->DR4);
 458                         if (ret)
 459                                 return ret;
 460                 }
 461
 462                 if (IS_I830(dev) || IS_845G(dev)) {
 463                         ret = intel_ring_begin(ring, 4);
 464                         if (ret)
 465                                 return ret;
 466
 467                         intel_ring_emit(ring, MI_BATCH_BUFFER);
 468                         intel_ring_emit(ring, exec_start | MI_BATCH_NON_SECURE);
 469                         intel_ring_emit(ring, exec_start + exec_len - 4);
 470                         intel_ring_emit(ring, 0);
 471                 } else {
 472                         ret = intel_ring_begin(ring, 2);
 473                         if (ret)
 474                                 return ret;
 475
 476                         if (INTEL_INFO(dev)->gen >= 4) {
 477                                 intel_ring_emit(ring,
 478                                                 MI_BATCH_BUFFER_START | (2 << 6)
 479                                                 | MI_BATCH_NON_SECURE_I965);
 480                                 intel_ring_emit(ring, exec_start);
 481                         } else {
 482                                 intel_ring_emit(ring, MI_BATCH_BUFFER_START
 483                                                 | (2 << 6));
 484                                 intel_ring_emit(ring, exec_start |
 485                                                 MI_BATCH_NON_SECURE);
 486                         }
 487                 }
 488                 intel_ring_advance(ring);
 489         }
 490
 491         if (IS_G4X(dev) || IS_GEN5(dev)) {
 492                 if (intel_ring_begin(ring, 2) == 0) {
 493                         intel_ring_emit(ring, MI_FLUSH |
 494                                         MI_NO_WRITE_FLUSH |
 495                                         MI_INVALIDATE_ISP );
 496                         intel_ring_emit(ring, MI_NOOP);
 497                         intel_ring_advance(ring);
 498                 }
 499         }
 500         /* XXX breadcrumb */
 501
 502         return 0;
 503 }
 504
 505 static void cleanup_status_page(struct intel_ring_buffer *ring)
 506 {
 507         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 508         struct drm_gem_object *obj;
 509         struct drm_i915_gem_object *obj_priv;
 510
 511         obj = ring->status_page.obj;
 512         if (obj == NULL)
 513                 return;
 514         obj_priv = to_intel_bo(obj);
 515
 516         kunmap(obj_priv->pages[0]);
 517         i915_gem_object_unpin(obj);
 518         drm_gem_object_unreference(obj);
 519         ring->status_page.obj = NULL;
 520
 521         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 522 }
 523
 524 static int init_status_page(struct intel_ring_buffer *ring)
 525 {
 526         struct drm_device *dev = ring->dev;
 527         drm_i915_private_t *dev_priv = dev->dev_private;
 528         struct drm_gem_object *obj;
 529         struct drm_i915_gem_object *obj_priv;
 530         int ret;
 531
 532         obj = i915_gem_alloc_object(dev, 4096);
 533         if (obj == NULL) {
 534                 DRM_ERROR("Failed to allocate status page\n");
 535                 ret = -ENOMEM;
 536                 goto err;
 537         }
 538         obj_priv = to_intel_bo(obj);
 539         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
 540
 541         ret = i915_gem_object_pin(obj, 4096);
 542         if (ret != 0) {
 543                 goto err_unref;
 544         }
 545
 546         ring->status_page.gfx_addr = obj_priv->gtt_offset;
 547         ring->status_page.page_addr = kmap(obj_priv->pages[0]);
 548         if (ring->status_page.page_addr == NULL) {
 549                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 550                 goto err_unpin;
 551         }
 552         ring->status_page.obj = obj;
 553         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 554
 555         intel_ring_setup_status_page(ring);
 556         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 557                         ring->name, ring->status_page.gfx_addr);
 558
 559         return 0;
 560
 561 err_unpin:
 562         i915_gem_object_unpin(obj);
 563 err_unref:
 564         drm_gem_object_unreference(obj);
 565 err:
 566         return ret;
 567 }
 568
 569 int intel_init_ring_buffer(struct drm_device *dev,
 570                            struct intel_ring_buffer *ring)
 571 {
 572         struct drm_i915_private *dev_priv = dev->dev_private;
 573         struct drm_i915_gem_object *obj_priv;
 574         struct drm_gem_object *obj;
 575         int ret;
 576
 577         ring->dev = dev;
 578         INIT_LIST_HEAD(&ring->active_list);
 579         INIT_LIST_HEAD(&ring->request_list);
 580         INIT_LIST_HEAD(&ring->gpu_write_list);
 581
 582         if (I915_NEED_GFX_HWS(dev)) {
 583                 ret = init_status_page(ring);
 584                 if (ret)
 585                         return ret;
 586         }
 587
 588         obj = i915_gem_alloc_object(dev, ring->size);
 589         if (obj == NULL) {
 590                 DRM_ERROR("Failed to allocate ringbuffer\n");
 591                 ret = -ENOMEM;
 592                 goto err_hws;
 593         }
 594
 595         ring->gem_object = obj;
 596
 597         ret = i915_gem_object_pin(obj, PAGE_SIZE);
 598         if (ret)
 599                 goto err_unref;
 600
 601         obj_priv = to_intel_bo(obj);
 602         ring->map.size = ring->size;
 603         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
 604         ring->map.type = 0;
 605         ring->map.flags = 0;
 606         ring->map.mtrr = 0;
 607
 608         drm_core_ioremap_wc(&ring->map, dev);
 609         if (ring->map.handle == NULL) {
 610                 DRM_ERROR("Failed to map ringbuffer.\n");
 611                 ret = -EINVAL;
 612                 goto err_unpin;
 613         }
 614
 615         ring->virtual_start = ring->map.handle;
 616         ret = ring->init(ring);
 617         if (ret)
 618                 goto err_unmap;
 619
 620         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 621                 i915_kernel_lost_context(dev);
 622         else {
 623                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 624                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 625                 ring->space = ring->head - (ring->tail + 8);
 626                 if (ring->space < 0)
 627                         ring->space += ring->size;
 628         }
 629         return ret;
 630
 631 err_unmap:
 632         drm_core_ioremapfree(&ring->map, dev);
 633 err_unpin:
 634         i915_gem_object_unpin(obj);
 635 err_unref:
 636         drm_gem_object_unreference(obj);
 637         ring->gem_object = NULL;
 638 err_hws:
 639         cleanup_status_page(ring);
 640         return ret;
 641 }
 642
 643 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 644 {
 645         if (ring->gem_object == NULL)
 646                 return;
 647
 648         drm_core_ioremapfree(&ring->map, ring->dev);
 649
 650         i915_gem_object_unpin(ring->gem_object);
 651         drm_gem_object_unreference(ring->gem_object);
 652         ring->gem_object = NULL;
 653
 654         cleanup_status_page(ring);
 655 }
 656
 657 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
 658 {
 659         unsigned int *virt;
 660         int rem;
 661         rem = ring->size - ring->tail;
 662
 663         if (ring->space < rem) {
 664                 int ret = intel_wait_ring_buffer(ring, rem);
 665                 if (ret)
 666                         return ret;
 667         }
 668
 669         virt = (unsigned int *)(ring->virtual_start + ring->tail);
 670         rem /= 8;
 671         while (rem--) {
 672                 *virt++ = MI_NOOP;
 673                 *virt++ = MI_NOOP;
 674         }
 675
 676         ring->tail = 0;
 677         ring->space = ring->head - 8;
 678
 679         return 0;
 680 }
 681
 682 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
 683 {
 684         struct drm_device *dev = ring->dev;
 685         drm_i915_private_t *dev_priv = dev->dev_private;
 686         unsigned long end;
 687
 688         trace_i915_ring_wait_begin (dev);
 689         end = jiffies + 3 * HZ;
 690         do {
 691                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 692                 ring->space = ring->head - (ring->tail + 8);
 693                 if (ring->space < 0)
 694                         ring->space += ring->size;
 695                 if (ring->space >= n) {
 696                         trace_i915_ring_wait_end(dev);
 697                         return 0;
 698                 }
 699
 700                 if (dev->primary->master) {
 701                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 702                         if (master_priv->sarea_priv)
 703                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 704                 }
 705
 706                 msleep(1);
 707         } while (!time_after(jiffies, end));
 708         trace_i915_ring_wait_end (dev);
 709         return -EBUSY;
 710 }
 711
 712 int intel_ring_begin(struct intel_ring_buffer *ring,
 713                      int num_dwords)
 714 {
 715         int n = 4*num_dwords;
 716         int ret;
 717
 718         if (unlikely(ring->tail + n > ring->size)) {
 719                 ret = intel_wrap_ring_buffer(ring);
 720                 if (unlikely(ret))
 721                         return ret;
 722         }
 723
 724         if (unlikely(ring->space < n)) {
 725                 ret = intel_wait_ring_buffer(ring, n);
 726                 if (unlikely(ret))
 727                         return ret;
 728         }
 729
 730         ring->space -= n;
 731         return 0;
 732 }
 733
 734 void intel_ring_advance(struct intel_ring_buffer *ring)
 735 {
 736         ring->tail &= ring->size - 1;
 737         ring->write_tail(ring, ring->tail);
 738 }
 739
 740 static const struct intel_ring_buffer render_ring = {
 741         .name                   = "render ring",
 742         .id                     = RING_RENDER,
 743         .mmio_base              = RENDER_RING_BASE,
 744         .size                   = 32 * PAGE_SIZE,
 745         .init                   = init_render_ring,
 746         .write_tail             = ring_write_tail,
 747         .flush                  = render_ring_flush,
 748         .add_request            = render_ring_add_request,
 749         .get_seqno              = render_ring_get_seqno,
 750         .user_irq_get           = render_ring_get_user_irq,
 751         .user_irq_put           = render_ring_put_user_irq,
 752         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
 753 };
 754
 755 /* ring buffer for bit-stream decoder */
 756
 757 static const struct intel_ring_buffer bsd_ring = {
 758         .name                   = "bsd ring",
 759         .id                     = RING_BSD,
 760         .mmio_base              = BSD_RING_BASE,
 761         .size                   = 32 * PAGE_SIZE,
 762         .init                   = init_ring_common,
 763         .write_tail             = ring_write_tail,
 764         .flush                  = bsd_ring_flush,
 765         .add_request            = ring_add_request,
 766         .get_seqno              = ring_status_page_get_seqno,
 767         .user_irq_get           = bsd_ring_get_user_irq,
 768         .user_irq_put           = bsd_ring_put_user_irq,
 769         .dispatch_execbuffer    = ring_dispatch_execbuffer,
 770 };
 771
 772
 773 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
 774                                      u32 value)
 775 {
 776        drm_i915_private_t *dev_priv = ring->dev->dev_private;
 777
 778        /* Every tail move must follow the sequence below */
 779        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
 780                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
 781                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
 782        I915_WRITE(GEN6_BSD_RNCID, 0x0);
 783
 784        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
 785                                GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
 786                        50))
 787                DRM_ERROR("timed out waiting for IDLE Indicator\n");
 788
 789        I915_WRITE_TAIL(ring, value);
 790        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
 791                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
 792                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
 793 }
 794
 795 static void gen6_ring_flush(struct intel_ring_buffer *ring,
 796                             u32 invalidate_domains,
 797                             u32 flush_domains)
 798 {
 799         if (intel_ring_begin(ring, 4) == 0) {
 800                 intel_ring_emit(ring, MI_FLUSH_DW);
 801                 intel_ring_emit(ring, 0);
 802                 intel_ring_emit(ring, 0);
 803                 intel_ring_emit(ring, 0);
 804                 intel_ring_advance(ring);
 805         }
 806 }
 807
 808 static int
 809 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 810                               struct drm_i915_gem_execbuffer2 *exec,
 811                               struct drm_clip_rect *cliprects,
 812                               uint64_t exec_offset)
 813 {
 814        uint32_t exec_start;
 815        int ret;
 816
 817        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 818
 819        ret = intel_ring_begin(ring, 2);
 820        if (ret)
 821                return ret;
 822
 823        intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
 824        /* bit0-7 is the length on GEN6+ */
 825        intel_ring_emit(ring, exec_start);
 826        intel_ring_advance(ring);
 827
 828        return 0;
 829 }
 830
 831 /* ring buffer for Video Codec for Gen6+ */
 832 static const struct intel_ring_buffer gen6_bsd_ring = {
 833        .name                    = "gen6 bsd ring",
 834        .id                      = RING_BSD,
 835        .mmio_base               = GEN6_BSD_RING_BASE,
 836        .size                    = 32 * PAGE_SIZE,
 837        .init                    = init_ring_common,
 838        .write_tail              = gen6_bsd_ring_write_tail,
 839        .flush                   = gen6_ring_flush,
 840        .add_request             = ring_add_request,
 841        .get_seqno               = ring_status_page_get_seqno,
 842        .user_irq_get            = bsd_ring_get_user_irq,
 843        .user_irq_put            = bsd_ring_put_user_irq,
 844        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
 845 };
 846
 847 /* Blitter support (SandyBridge+) */
 848
 849 static void
 850 blt_ring_get_user_irq(struct intel_ring_buffer *ring)
 851 {
 852         /* do nothing */
 853 }
 854 static void
 855 blt_ring_put_user_irq(struct intel_ring_buffer *ring)
 856 {
 857         /* do nothing */
 858 }
 859
 860 static const struct intel_ring_buffer gen6_blt_ring = {
 861        .name                    = "blt ring",
 862        .id                      = RING_BLT,
 863        .mmio_base               = BLT_RING_BASE,
 864        .size                    = 32 * PAGE_SIZE,
 865        .init                    = init_ring_common,
 866        .write_tail              = ring_write_tail,
 867        .flush                   = gen6_ring_flush,
 868        .add_request             = ring_add_request,
 869        .get_seqno               = ring_status_page_get_seqno,
 870        .user_irq_get            = blt_ring_get_user_irq,
 871        .user_irq_put            = blt_ring_put_user_irq,
 872        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
 873 };
 874
 875 int intel_init_render_ring_buffer(struct drm_device *dev)
 876 {
 877         drm_i915_private_t *dev_priv = dev->dev_private;
 878
 879         dev_priv->render_ring = render_ring;
 880
 881         if (!I915_NEED_GFX_HWS(dev)) {
 882                 dev_priv->render_ring.status_page.page_addr
 883                         = dev_priv->status_page_dmah->vaddr;
 884                 memset(dev_priv->render_ring.status_page.page_addr,
 885                                 0, PAGE_SIZE);
 886         }
 887
 888         return intel_init_ring_buffer(dev, &dev_priv->render_ring);
 889 }
 890
 891 int intel_init_bsd_ring_buffer(struct drm_device *dev)
 892 {
 893         drm_i915_private_t *dev_priv = dev->dev_private;
 894
 895         if (IS_GEN6(dev))
 896                 dev_priv->bsd_ring = gen6_bsd_ring;
 897         else
 898                 dev_priv->bsd_ring = bsd_ring;
 899
 900         return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
 901 }
 902
 903 int intel_init_blt_ring_buffer(struct drm_device *dev)
 904 {
 905         drm_i915_private_t *dev_priv = dev->dev_private;
 906
 907         dev_priv->blt_ring = gen6_blt_ring;
 908
 909         return intel_init_ring_buffer(dev, &dev_priv->blt_ring);
 910 }