drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35
  36 static u32 i915_gem_get_seqno(struct drm_device *dev)
  37 {
  38         drm_i915_private_t *dev_priv = dev->dev_private;
  39         u32 seqno;
  40
  41         seqno = dev_priv->next_seqno;
  42
  43         /* reserve 0 for non-seqno */
  44         if (++dev_priv->next_seqno == 0)
  45                 dev_priv->next_seqno = 1;
  46
  47         return seqno;
  48 }
  49
  50 static void
  51 render_ring_flush(struct drm_device *dev,
  52                 struct intel_ring_buffer *ring,
  53                 u32     invalidate_domains,
  54                 u32     flush_domains)
  55 {
  56         drm_i915_private_t *dev_priv = dev->dev_private;
  57         u32 cmd;
  58
  59 #if WATCH_EXEC
  60         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
  61                   invalidate_domains, flush_domains);
  62 #endif
  63
  64         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
  65                                      invalidate_domains, flush_domains);
  66
  67         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
  68                 /*
  69                  * read/write caches:
  70                  *
  71                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  72                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  73                  * also flushed at 2d versus 3d pipeline switches.
  74                  *
  75                  * read-only caches:
  76                  *
  77                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  78                  * MI_READ_FLUSH is set, and is always flushed on 965.
  79                  *
  80                  * I915_GEM_DOMAIN_COMMAND may not exist?
  81                  *
  82                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  83                  * invalidated when MI_EXE_FLUSH is set.
  84                  *
  85                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  86                  * invalidated with every MI_FLUSH.
  87                  *
  88                  * TLBs:
  89                  *
  90                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  91                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  92                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  93                  * are flushed at any MI_FLUSH.
  94                  */
  95
  96                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  97                 if ((invalidate_domains|flush_domains) &
  98                     I915_GEM_DOMAIN_RENDER)
  99                         cmd &= ~MI_NO_WRITE_FLUSH;
 100                 if (!IS_I965G(dev)) {
 101                         /*
 102                          * On the 965, the sampler cache always gets flushed
 103                          * and this bit is reserved.
 104                          */
 105                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 106                                 cmd |= MI_READ_FLUSH;
 107                 }
 108                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 109                         cmd |= MI_EXE_FLUSH;
 110
 111 #if WATCH_EXEC
 112                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 113 #endif
 114                 intel_ring_begin(dev, ring, 2);
 115                 intel_ring_emit(dev, ring, cmd);
 116                 intel_ring_emit(dev, ring, MI_NOOP);
 117                 intel_ring_advance(dev, ring);
 118         }
 119 }
 120
 121 static unsigned int render_ring_get_head(struct drm_device *dev,
 122                 struct intel_ring_buffer *ring)
 123 {
 124         drm_i915_private_t *dev_priv = dev->dev_private;
 125         return I915_READ(PRB0_HEAD) & HEAD_ADDR;
 126 }
 127
 128 static unsigned int render_ring_get_tail(struct drm_device *dev,
 129                 struct intel_ring_buffer *ring)
 130 {
 131         drm_i915_private_t *dev_priv = dev->dev_private;
 132         return I915_READ(PRB0_TAIL) & TAIL_ADDR;
 133 }
 134
 135 static unsigned int render_ring_get_active_head(struct drm_device *dev,
 136                 struct intel_ring_buffer *ring)
 137 {
 138         drm_i915_private_t *dev_priv = dev->dev_private;
 139         u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
 140
 141         return I915_READ(acthd_reg);
 142 }
 143
 144 static void render_ring_advance_ring(struct drm_device *dev,
 145                 struct intel_ring_buffer *ring)
 146 {
 147         drm_i915_private_t *dev_priv = dev->dev_private;
 148         I915_WRITE(PRB0_TAIL, ring->tail);
 149 }
 150
 151 static int init_ring_common(struct drm_device *dev,
 152                 struct intel_ring_buffer *ring)
 153 {
 154         u32 head;
 155         drm_i915_private_t *dev_priv = dev->dev_private;
 156         struct drm_i915_gem_object *obj_priv;
 157         obj_priv = to_intel_bo(ring->gem_object);
 158
 159         /* Stop the ring if it's running. */
 160         I915_WRITE(ring->regs.ctl, 0);
 161         I915_WRITE(ring->regs.head, 0);
 162         I915_WRITE(ring->regs.tail, 0);
 163
 164         /* Initialize the ring. */
 165         I915_WRITE(ring->regs.start, obj_priv->gtt_offset);
 166         head = ring->get_head(dev, ring);
 167
 168         /* G45 ring initialization fails to reset head to zero */
 169         if (head != 0) {
 170                 DRM_ERROR("%s head not reset to zero "
 171                                 "ctl %08x head %08x tail %08x start %08x\n",
 172                                 ring->name,
 173                                 I915_READ(ring->regs.ctl),
 174                                 I915_READ(ring->regs.head),
 175                                 I915_READ(ring->regs.tail),
 176                                 I915_READ(ring->regs.start));
 177
 178                 I915_WRITE(ring->regs.head, 0);
 179
 180                 DRM_ERROR("%s head forced to zero "
 181                                 "ctl %08x head %08x tail %08x start %08x\n",
 182                                 ring->name,
 183                                 I915_READ(ring->regs.ctl),
 184                                 I915_READ(ring->regs.head),
 185                                 I915_READ(ring->regs.tail),
 186                                 I915_READ(ring->regs.start));
 187         }
 188
 189         I915_WRITE(ring->regs.ctl,
 190                         ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
 191                         | RING_NO_REPORT | RING_VALID);
 192
 193         head = I915_READ(ring->regs.head) & HEAD_ADDR;
 194         /* If the head is still not zero, the ring is dead */
 195         if (head != 0) {
 196                 DRM_ERROR("%s initialization failed "
 197                                 "ctl %08x head %08x tail %08x start %08x\n",
 198                                 ring->name,
 199                                 I915_READ(ring->regs.ctl),
 200                                 I915_READ(ring->regs.head),
 201                                 I915_READ(ring->regs.tail),
 202                                 I915_READ(ring->regs.start));
 203                 return -EIO;
 204         }
 205
 206         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 207                 i915_kernel_lost_context(dev);
 208         else {
 209                 ring->head = ring->get_head(dev, ring);
 210                 ring->tail = ring->get_tail(dev, ring);
 211                 ring->space = ring->head - (ring->tail + 8);
 212                 if (ring->space < 0)
 213                         ring->space += ring->size;
 214         }
 215         return 0;
 216 }
 217
 218 static int init_render_ring(struct drm_device *dev,
 219                 struct intel_ring_buffer *ring)
 220 {
 221         drm_i915_private_t *dev_priv = dev->dev_private;
 222         int ret = init_ring_common(dev, ring);
 223         if (IS_I9XX(dev) && !IS_GEN3(dev)) {
 224                 I915_WRITE(MI_MODE,
 225                                 (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
 226         }
 227         return ret;
 228 }
 229
 230 #define PIPE_CONTROL_FLUSH(addr)                                        \
 231 do {                                                                    \
 232         OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
 233                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
 234         OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
 235         OUT_RING(0);                                                    \
 236         OUT_RING(0);                                                    \
 237 } while (0)
 238
 239 /**
 240  * Creates a new sequence number, emitting a write of it to the status page
 241  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 242  *
 243  * Must be called with struct_lock held.
 244  *
 245  * Returned sequence numbers are nonzero on success.
 246  */
 247 static u32
 248 render_ring_add_request(struct drm_device *dev,
 249                 struct intel_ring_buffer *ring,
 250                 struct drm_file *file_priv,
 251                 u32 flush_domains)
 252 {
 253         drm_i915_private_t *dev_priv = dev->dev_private;
 254         u32 seqno;
 255
 256         seqno = i915_gem_get_seqno(dev);
 257
 258         if (IS_GEN6(dev)) {
 259                 BEGIN_LP_RING(6);
 260                 OUT_RING(GFX_OP_PIPE_CONTROL | 3);
 261                 OUT_RING(PIPE_CONTROL_QW_WRITE |
 262                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
 263                          PIPE_CONTROL_NOTIFY);
 264                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 265                 OUT_RING(seqno);
 266                 OUT_RING(0);
 267                 OUT_RING(0);
 268                 ADVANCE_LP_RING();
 269         } else if (HAS_PIPE_CONTROL(dev)) {
 270                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
 271
 272                 /*
 273                  * Workaround qword write incoherence by flushing the
 274                  * PIPE_NOTIFY buffers out to memory before requesting
 275                  * an interrupt.
 276                  */
 277                 BEGIN_LP_RING(32);
 278                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 279                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
 280                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 281                 OUT_RING(seqno);
 282                 OUT_RING(0);
 283                 PIPE_CONTROL_FLUSH(scratch_addr);
 284                 scratch_addr += 128; /* write to separate cachelines */
 285                 PIPE_CONTROL_FLUSH(scratch_addr);
 286                 scratch_addr += 128;
 287                 PIPE_CONTROL_FLUSH(scratch_addr);
 288                 scratch_addr += 128;
 289                 PIPE_CONTROL_FLUSH(scratch_addr);
 290                 scratch_addr += 128;
 291                 PIPE_CONTROL_FLUSH(scratch_addr);
 292                 scratch_addr += 128;
 293                 PIPE_CONTROL_FLUSH(scratch_addr);
 294                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 295                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
 296                          PIPE_CONTROL_NOTIFY);
 297                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 298                 OUT_RING(seqno);
 299                 OUT_RING(0);
 300                 ADVANCE_LP_RING();
 301         } else {
 302                 BEGIN_LP_RING(4);
 303                 OUT_RING(MI_STORE_DWORD_INDEX);
 304                 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 305                 OUT_RING(seqno);
 306
 307                 OUT_RING(MI_USER_INTERRUPT);
 308                 ADVANCE_LP_RING();
 309         }
 310         return seqno;
 311 }
 312
 313 static u32
 314 render_ring_get_gem_seqno(struct drm_device *dev,
 315                 struct intel_ring_buffer *ring)
 316 {
 317         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 318         if (HAS_PIPE_CONTROL(dev))
 319                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
 320         else
 321                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 322 }
 323
 324 static void
 325 render_ring_get_user_irq(struct drm_device *dev,
 326                 struct intel_ring_buffer *ring)
 327 {
 328         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 329         unsigned long irqflags;
 330
 331         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 332         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
 333                 if (HAS_PCH_SPLIT(dev))
 334                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 335                 else
 336                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 337         }
 338         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 339 }
 340
 341 static void
 342 render_ring_put_user_irq(struct drm_device *dev,
 343                 struct intel_ring_buffer *ring)
 344 {
 345         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 346         unsigned long irqflags;
 347
 348         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 349         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
 350         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
 351                 if (HAS_PCH_SPLIT(dev))
 352                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 353                 else
 354                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 355         }
 356         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 357 }
 358
 359 static void render_setup_status_page(struct drm_device *dev,
 360         struct  intel_ring_buffer *ring)
 361 {
 362         drm_i915_private_t *dev_priv = dev->dev_private;
 363         if (IS_GEN6(dev)) {
 364                 I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr);
 365                 I915_READ(HWS_PGA_GEN6); /* posting read */
 366         } else {
 367                 I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
 368                 I915_READ(HWS_PGA); /* posting read */
 369         }
 370
 371 }
 372
 373 void
 374 bsd_ring_flush(struct drm_device *dev,
 375                 struct intel_ring_buffer *ring,
 376                 u32     invalidate_domains,
 377                 u32     flush_domains)
 378 {
 379         intel_ring_begin(dev, ring, 2);
 380         intel_ring_emit(dev, ring, MI_FLUSH);
 381         intel_ring_emit(dev, ring, MI_NOOP);
 382         intel_ring_advance(dev, ring);
 383 }
 384
 385 static inline unsigned int bsd_ring_get_head(struct drm_device *dev,
 386                 struct intel_ring_buffer *ring)
 387 {
 388         drm_i915_private_t *dev_priv = dev->dev_private;
 389         return I915_READ(BSD_RING_HEAD) & HEAD_ADDR;
 390 }
 391
 392 static inline unsigned int bsd_ring_get_tail(struct drm_device *dev,
 393                 struct intel_ring_buffer *ring)
 394 {
 395         drm_i915_private_t *dev_priv = dev->dev_private;
 396         return I915_READ(BSD_RING_TAIL) & TAIL_ADDR;
 397 }
 398
 399 static inline unsigned int bsd_ring_get_active_head(struct drm_device *dev,
 400                 struct intel_ring_buffer *ring)
 401 {
 402         drm_i915_private_t *dev_priv = dev->dev_private;
 403         return I915_READ(BSD_RING_ACTHD);
 404 }
 405
 406 static inline void bsd_ring_advance_ring(struct drm_device *dev,
 407                 struct intel_ring_buffer *ring)
 408 {
 409         drm_i915_private_t *dev_priv = dev->dev_private;
 410         I915_WRITE(BSD_RING_TAIL, ring->tail);
 411 }
 412
 413 static int init_bsd_ring(struct drm_device *dev,
 414                 struct intel_ring_buffer *ring)
 415 {
 416         return init_ring_common(dev, ring);
 417 }
 418
 419 static u32
 420 bsd_ring_add_request(struct drm_device *dev,
 421                 struct intel_ring_buffer *ring,
 422                 struct drm_file *file_priv,
 423                 u32 flush_domains)
 424 {
 425         u32 seqno;
 426
 427         seqno = i915_gem_get_seqno(dev);
 428
 429         intel_ring_begin(dev, ring, 4);
 430         intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
 431         intel_ring_emit(dev, ring,
 432                         I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 433         intel_ring_emit(dev, ring, seqno);
 434         intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
 435         intel_ring_advance(dev, ring);
 436
 437         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
 438
 439         return seqno;
 440 }
 441
 442 static void bsd_setup_status_page(struct drm_device *dev,
 443                 struct  intel_ring_buffer *ring)
 444 {
 445         drm_i915_private_t *dev_priv = dev->dev_private;
 446         I915_WRITE(BSD_HWS_PGA, ring->status_page.gfx_addr);
 447         I915_READ(BSD_HWS_PGA);
 448 }
 449
 450 static void
 451 bsd_ring_get_user_irq(struct drm_device *dev,
 452                 struct intel_ring_buffer *ring)
 453 {
 454         /* do nothing */
 455 }
 456 static void
 457 bsd_ring_put_user_irq(struct drm_device *dev,
 458                 struct intel_ring_buffer *ring)
 459 {
 460         /* do nothing */
 461 }
 462
 463 static u32
 464 bsd_ring_get_gem_seqno(struct drm_device *dev,
 465                 struct intel_ring_buffer *ring)
 466 {
 467         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 468 }
 469
 470 static int
 471 bsd_ring_dispatch_gem_execbuffer(struct drm_device *dev,
 472                 struct intel_ring_buffer *ring,
 473                 struct drm_i915_gem_execbuffer2 *exec,
 474                 struct drm_clip_rect *cliprects,
 475                 uint64_t exec_offset)
 476 {
 477         uint32_t exec_start;
 478         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 479         intel_ring_begin(dev, ring, 2);
 480         intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START |
 481                         (2 << 6) | MI_BATCH_NON_SECURE_I965);
 482         intel_ring_emit(dev, ring, exec_start);
 483         intel_ring_advance(dev, ring);
 484         return 0;
 485 }
 486
 487
 488 static int
 489 render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
 490                 struct intel_ring_buffer *ring,
 491                 struct drm_i915_gem_execbuffer2 *exec,
 492                 struct drm_clip_rect *cliprects,
 493                 uint64_t exec_offset)
 494 {
 495         drm_i915_private_t *dev_priv = dev->dev_private;
 496         int nbox = exec->num_cliprects;
 497         int i = 0, count;
 498         uint32_t exec_start, exec_len;
 499         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 500         exec_len = (uint32_t) exec->batch_len;
 501
 502         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
 503
 504         count = nbox ? nbox : 1;
 505
 506         for (i = 0; i < count; i++) {
 507                 if (i < nbox) {
 508                         int ret = i915_emit_box(dev, cliprects, i,
 509                                                 exec->DR1, exec->DR4);
 510                         if (ret)
 511                                 return ret;
 512                 }
 513
 514                 if (IS_I830(dev) || IS_845G(dev)) {
 515                         intel_ring_begin(dev, ring, 4);
 516                         intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
 517                         intel_ring_emit(dev, ring,
 518                                         exec_start | MI_BATCH_NON_SECURE);
 519                         intel_ring_emit(dev, ring, exec_start + exec_len - 4);
 520                         intel_ring_emit(dev, ring, 0);
 521                 } else {
 522                         intel_ring_begin(dev, ring, 4);
 523                         if (IS_I965G(dev)) {
 524                                 intel_ring_emit(dev, ring,
 525                                                 MI_BATCH_BUFFER_START | (2 << 6)
 526                                                 | MI_BATCH_NON_SECURE_I965);
 527                                 intel_ring_emit(dev, ring, exec_start);
 528                         } else {
 529                                 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
 530                                                 | (2 << 6));
 531                                 intel_ring_emit(dev, ring, exec_start |
 532                                                 MI_BATCH_NON_SECURE);
 533                         }
 534                 }
 535                 intel_ring_advance(dev, ring);
 536         }
 537
 538         if (IS_G4X(dev) || IS_IRONLAKE(dev)) {
 539                 intel_ring_begin(dev, ring, 2);
 540                 intel_ring_emit(dev, ring, MI_FLUSH |
 541                                 MI_NO_WRITE_FLUSH |
 542                                 MI_INVALIDATE_ISP );
 543                 intel_ring_emit(dev, ring, MI_NOOP);
 544                 intel_ring_advance(dev, ring);
 545         }
 546         /* XXX breadcrumb */
 547
 548         return 0;
 549 }
 550
 551 static void cleanup_status_page(struct drm_device *dev,
 552                 struct intel_ring_buffer *ring)
 553 {
 554         drm_i915_private_t *dev_priv = dev->dev_private;
 555         struct drm_gem_object *obj;
 556         struct drm_i915_gem_object *obj_priv;
 557
 558         obj = ring->status_page.obj;
 559         if (obj == NULL)
 560                 return;
 561         obj_priv = to_intel_bo(obj);
 562
 563         kunmap(obj_priv->pages[0]);
 564         i915_gem_object_unpin(obj);
 565         drm_gem_object_unreference(obj);
 566         ring->status_page.obj = NULL;
 567
 568         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 569 }
 570
 571 static int init_status_page(struct drm_device *dev,
 572                 struct intel_ring_buffer *ring)
 573 {
 574         drm_i915_private_t *dev_priv = dev->dev_private;
 575         struct drm_gem_object *obj;
 576         struct drm_i915_gem_object *obj_priv;
 577         int ret;
 578
 579         obj = i915_gem_alloc_object(dev, 4096);
 580         if (obj == NULL) {
 581                 DRM_ERROR("Failed to allocate status page\n");
 582                 ret = -ENOMEM;
 583                 goto err;
 584         }
 585         obj_priv = to_intel_bo(obj);
 586         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
 587
 588         ret = i915_gem_object_pin(obj, 4096);
 589         if (ret != 0) {
 590                 goto err_unref;
 591         }
 592
 593         ring->status_page.gfx_addr = obj_priv->gtt_offset;
 594         ring->status_page.page_addr = kmap(obj_priv->pages[0]);
 595         if (ring->status_page.page_addr == NULL) {
 596                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 597                 goto err_unpin;
 598         }
 599         ring->status_page.obj = obj;
 600         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 601
 602         ring->setup_status_page(dev, ring);
 603         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 604                         ring->name, ring->status_page.gfx_addr);
 605
 606         return 0;
 607
 608 err_unpin:
 609         i915_gem_object_unpin(obj);
 610 err_unref:
 611         drm_gem_object_unreference(obj);
 612 err:
 613         return ret;
 614 }
 615
 616
 617 int intel_init_ring_buffer(struct drm_device *dev,
 618                 struct intel_ring_buffer *ring)
 619 {
 620         struct drm_i915_gem_object *obj_priv;
 621         struct drm_gem_object *obj;
 622         int ret;
 623
 624         ring->dev = dev;
 625
 626         if (I915_NEED_GFX_HWS(dev)) {
 627                 ret = init_status_page(dev, ring);
 628                 if (ret)
 629                         return ret;
 630         }
 631
 632         obj = i915_gem_alloc_object(dev, ring->size);
 633         if (obj == NULL) {
 634                 DRM_ERROR("Failed to allocate ringbuffer\n");
 635                 ret = -ENOMEM;
 636                 goto err_hws;
 637         }
 638
 639         ring->gem_object = obj;
 640
 641         ret = i915_gem_object_pin(obj, ring->alignment);
 642         if (ret)
 643                 goto err_unref;
 644
 645         obj_priv = to_intel_bo(obj);
 646         ring->map.size = ring->size;
 647         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
 648         ring->map.type = 0;
 649         ring->map.flags = 0;
 650         ring->map.mtrr = 0;
 651
 652         drm_core_ioremap_wc(&ring->map, dev);
 653         if (ring->map.handle == NULL) {
 654                 DRM_ERROR("Failed to map ringbuffer.\n");
 655                 ret = -EINVAL;
 656                 goto err_unpin;
 657         }
 658
 659         ring->virtual_start = ring->map.handle;
 660         ret = ring->init(dev, ring);
 661         if (ret)
 662                 goto err_unmap;
 663
 664         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 665                 i915_kernel_lost_context(dev);
 666         else {
 667                 ring->head = ring->get_head(dev, ring);
 668                 ring->tail = ring->get_tail(dev, ring);
 669                 ring->space = ring->head - (ring->tail + 8);
 670                 if (ring->space < 0)
 671                         ring->space += ring->size;
 672         }
 673         INIT_LIST_HEAD(&ring->active_list);
 674         INIT_LIST_HEAD(&ring->request_list);
 675         return ret;
 676
 677 err_unmap:
 678         drm_core_ioremapfree(&ring->map, dev);
 679 err_unpin:
 680         i915_gem_object_unpin(obj);
 681 err_unref:
 682         drm_gem_object_unreference(obj);
 683         ring->gem_object = NULL;
 684 err_hws:
 685         cleanup_status_page(dev, ring);
 686         return ret;
 687 }
 688
 689 void intel_cleanup_ring_buffer(struct drm_device *dev,
 690                 struct intel_ring_buffer *ring)
 691 {
 692         if (ring->gem_object == NULL)
 693                 return;
 694
 695         drm_core_ioremapfree(&ring->map, dev);
 696
 697         i915_gem_object_unpin(ring->gem_object);
 698         drm_gem_object_unreference(ring->gem_object);
 699         ring->gem_object = NULL;
 700         cleanup_status_page(dev, ring);
 701 }
 702
 703 int intel_wrap_ring_buffer(struct drm_device *dev,
 704                 struct intel_ring_buffer *ring)
 705 {
 706         unsigned int *virt;
 707         int rem;
 708         rem = ring->size - ring->tail;
 709
 710         if (ring->space < rem) {
 711                 int ret = intel_wait_ring_buffer(dev, ring, rem);
 712                 if (ret)
 713                         return ret;
 714         }
 715
 716         virt = (unsigned int *)(ring->virtual_start + ring->tail);
 717         rem /= 8;
 718         while (rem--) {
 719                 *virt++ = MI_NOOP;
 720                 *virt++ = MI_NOOP;
 721         }
 722
 723         ring->tail = 0;
 724         ring->space = ring->head - 8;
 725
 726         return 0;
 727 }
 728
 729 int intel_wait_ring_buffer(struct drm_device *dev,
 730                 struct intel_ring_buffer *ring, int n)
 731 {
 732         unsigned long end;
 733
 734         trace_i915_ring_wait_begin (dev);
 735         end = jiffies + 3 * HZ;
 736         do {
 737                 ring->head = ring->get_head(dev, ring);
 738                 ring->space = ring->head - (ring->tail + 8);
 739                 if (ring->space < 0)
 740                         ring->space += ring->size;
 741                 if (ring->space >= n) {
 742                         trace_i915_ring_wait_end (dev);
 743                         return 0;
 744                 }
 745
 746                 if (dev->primary->master) {
 747                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 748                         if (master_priv->sarea_priv)
 749                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 750                 }
 751
 752                 yield();
 753         } while (!time_after(jiffies, end));
 754         trace_i915_ring_wait_end (dev);
 755         return -EBUSY;
 756 }
 757
 758 void intel_ring_begin(struct drm_device *dev,
 759                 struct intel_ring_buffer *ring, int num_dwords)
 760 {
 761         int n = 4*num_dwords;
 762         if (unlikely(ring->tail + n > ring->size))
 763                 intel_wrap_ring_buffer(dev, ring);
 764         if (unlikely(ring->space < n))
 765                 intel_wait_ring_buffer(dev, ring, n);
 766
 767         ring->space -= n;
 768 }
 769
 770 void intel_ring_advance(struct drm_device *dev,
 771                 struct intel_ring_buffer *ring)
 772 {
 773         ring->tail &= ring->size - 1;
 774         ring->advance_ring(dev, ring);
 775 }
 776
 777 void intel_fill_struct(struct drm_device *dev,
 778                 struct intel_ring_buffer *ring,
 779                 void *data,
 780                 unsigned int len)
 781 {
 782         unsigned int *virt = ring->virtual_start + ring->tail;
 783         BUG_ON((len&~(4-1)) != 0);
 784         intel_ring_begin(dev, ring, len/4);
 785         memcpy(virt, data, len);
 786         ring->tail += len;
 787         ring->tail &= ring->size - 1;
 788         ring->space -= len;
 789         intel_ring_advance(dev, ring);
 790 }
 791
 792 struct intel_ring_buffer render_ring = {
 793         .name                   = "render ring",
 794         .regs                   = {
 795                 .ctl = PRB0_CTL,
 796                 .head = PRB0_HEAD,
 797                 .tail = PRB0_TAIL,
 798                 .start = PRB0_START
 799         },
 800         .ring_flag              = I915_EXEC_RENDER,
 801         .size                   = 32 * PAGE_SIZE,
 802         .alignment              = PAGE_SIZE,
 803         .virtual_start          = NULL,
 804         .dev                    = NULL,
 805         .gem_object             = NULL,
 806         .head                   = 0,
 807         .tail                   = 0,
 808         .space                  = 0,
 809         .user_irq_refcount      = 0,
 810         .irq_gem_seqno          = 0,
 811         .waiting_gem_seqno      = 0,
 812         .setup_status_page      = render_setup_status_page,
 813         .init                   = init_render_ring,
 814         .get_head               = render_ring_get_head,
 815         .get_tail               = render_ring_get_tail,
 816         .get_active_head        = render_ring_get_active_head,
 817         .advance_ring           = render_ring_advance_ring,
 818         .flush                  = render_ring_flush,
 819         .add_request            = render_ring_add_request,
 820         .get_gem_seqno          = render_ring_get_gem_seqno,
 821         .user_irq_get           = render_ring_get_user_irq,
 822         .user_irq_put           = render_ring_put_user_irq,
 823         .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
 824         .status_page            = {NULL, 0, NULL},
 825         .map                    = {0,}
 826 };
 827
 828 /* ring buffer for bit-stream decoder */
 829
 830 struct intel_ring_buffer bsd_ring = {
 831         .name                   = "bsd ring",
 832         .regs                   = {
 833                 .ctl = BSD_RING_CTL,
 834                 .head = BSD_RING_HEAD,
 835                 .tail = BSD_RING_TAIL,
 836                 .start = BSD_RING_START
 837         },
 838         .ring_flag              = I915_EXEC_BSD,
 839         .size                   = 32 * PAGE_SIZE,
 840         .alignment              = PAGE_SIZE,
 841         .virtual_start          = NULL,
 842         .dev                    = NULL,
 843         .gem_object             = NULL,
 844         .head                   = 0,
 845         .tail                   = 0,
 846         .space                  = 0,
 847         .user_irq_refcount      = 0,
 848         .irq_gem_seqno          = 0,
 849         .waiting_gem_seqno      = 0,
 850         .setup_status_page      = bsd_setup_status_page,
 851         .init                   = init_bsd_ring,
 852         .get_head               = bsd_ring_get_head,
 853         .get_tail               = bsd_ring_get_tail,
 854         .get_active_head        = bsd_ring_get_active_head,
 855         .advance_ring           = bsd_ring_advance_ring,
 856         .flush                  = bsd_ring_flush,
 857         .add_request            = bsd_ring_add_request,
 858         .get_gem_seqno          = bsd_ring_get_gem_seqno,
 859         .user_irq_get           = bsd_ring_get_user_irq,
 860         .user_irq_put           = bsd_ring_put_user_irq,
 861         .dispatch_gem_execbuffer = bsd_ring_dispatch_gem_execbuffer,
 862         .status_page            = {NULL, 0, NULL},
 863         .map                    = {0,}
 864 };