drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35
  36 static u32 i915_gem_get_seqno(struct drm_device *dev)
  37 {
  38         drm_i915_private_t *dev_priv = dev->dev_private;
  39         u32 seqno;
  40
  41         seqno = dev_priv->next_seqno;
  42
  43         /* reserve 0 for non-seqno */
  44         if (++dev_priv->next_seqno == 0)
  45                 dev_priv->next_seqno = 1;
  46
  47         return seqno;
  48 }
  49
  50 static void
  51 render_ring_flush(struct drm_device *dev,
  52                 struct intel_ring_buffer *ring,
  53                 u32     invalidate_domains,
  54                 u32     flush_domains)
  55 {
  56         drm_i915_private_t *dev_priv = dev->dev_private;
  57         u32 cmd;
  58
  59 #if WATCH_EXEC
  60         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
  61                   invalidate_domains, flush_domains);
  62 #endif
  63
  64         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
  65                                      invalidate_domains, flush_domains);
  66
  67         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
  68                 /*
  69                  * read/write caches:
  70                  *
  71                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  72                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  73                  * also flushed at 2d versus 3d pipeline switches.
  74                  *
  75                  * read-only caches:
  76                  *
  77                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  78                  * MI_READ_FLUSH is set, and is always flushed on 965.
  79                  *
  80                  * I915_GEM_DOMAIN_COMMAND may not exist?
  81                  *
  82                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  83                  * invalidated when MI_EXE_FLUSH is set.
  84                  *
  85                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  86                  * invalidated with every MI_FLUSH.
  87                  *
  88                  * TLBs:
  89                  *
  90                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  91                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  92                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  93                  * are flushed at any MI_FLUSH.
  94                  */
  95
  96                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  97                 if ((invalidate_domains|flush_domains) &
  98                     I915_GEM_DOMAIN_RENDER)
  99                         cmd &= ~MI_NO_WRITE_FLUSH;
 100                 if (!IS_I965G(dev)) {
 101                         /*
 102                          * On the 965, the sampler cache always gets flushed
 103                          * and this bit is reserved.
 104                          */
 105                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 106                                 cmd |= MI_READ_FLUSH;
 107                 }
 108                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 109                         cmd |= MI_EXE_FLUSH;
 110
 111 #if WATCH_EXEC
 112                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 113 #endif
 114                 intel_ring_begin(dev, ring, 2);
 115                 intel_ring_emit(dev, ring, cmd);
 116                 intel_ring_emit(dev, ring, MI_NOOP);
 117                 intel_ring_advance(dev, ring);
 118         }
 119 }
 120
 121 static unsigned int render_ring_get_head(struct drm_device *dev,
 122                 struct intel_ring_buffer *ring)
 123 {
 124         drm_i915_private_t *dev_priv = dev->dev_private;
 125         return I915_READ(PRB0_HEAD) & HEAD_ADDR;
 126 }
 127
 128 static unsigned int render_ring_get_tail(struct drm_device *dev,
 129                 struct intel_ring_buffer *ring)
 130 {
 131         drm_i915_private_t *dev_priv = dev->dev_private;
 132         return I915_READ(PRB0_TAIL) & TAIL_ADDR;
 133 }
 134
 135 static unsigned int render_ring_get_active_head(struct drm_device *dev,
 136                 struct intel_ring_buffer *ring)
 137 {
 138         drm_i915_private_t *dev_priv = dev->dev_private;
 139         u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
 140
 141         return I915_READ(acthd_reg);
 142 }
 143
 144 static void render_ring_advance_ring(struct drm_device *dev,
 145                 struct intel_ring_buffer *ring)
 146 {
 147         drm_i915_private_t *dev_priv = dev->dev_private;
 148         I915_WRITE(PRB0_TAIL, ring->tail);
 149 }
 150
 151 static int init_ring_common(struct drm_device *dev,
 152                 struct intel_ring_buffer *ring)
 153 {
 154         u32 head;
 155         drm_i915_private_t *dev_priv = dev->dev_private;
 156         struct drm_i915_gem_object *obj_priv;
 157         obj_priv = to_intel_bo(ring->gem_object);
 158
 159         /* Stop the ring if it's running. */
 160         I915_WRITE(ring->regs.ctl, 0);
 161         I915_WRITE(ring->regs.head, 0);
 162         I915_WRITE(ring->regs.tail, 0);
 163
 164         /* Initialize the ring. */
 165         I915_WRITE(ring->regs.start, obj_priv->gtt_offset);
 166         head = ring->get_head(dev, ring);
 167
 168         /* G45 ring initialization fails to reset head to zero */
 169         if (head != 0) {
 170                 DRM_ERROR("%s head not reset to zero "
 171                                 "ctl %08x head %08x tail %08x start %08x\n",
 172                                 ring->name,
 173                                 I915_READ(ring->regs.ctl),
 174                                 I915_READ(ring->regs.head),
 175                                 I915_READ(ring->regs.tail),
 176                                 I915_READ(ring->regs.start));
 177
 178                 I915_WRITE(ring->regs.head, 0);
 179
 180                 DRM_ERROR("%s head forced to zero "
 181                                 "ctl %08x head %08x tail %08x start %08x\n",
 182                                 ring->name,
 183                                 I915_READ(ring->regs.ctl),
 184                                 I915_READ(ring->regs.head),
 185                                 I915_READ(ring->regs.tail),
 186                                 I915_READ(ring->regs.start));
 187         }
 188
 189         I915_WRITE(ring->regs.ctl,
 190                         ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
 191                         | RING_NO_REPORT | RING_VALID);
 192
 193         head = I915_READ(ring->regs.head) & HEAD_ADDR;
 194         /* If the head is still not zero, the ring is dead */
 195         if (head != 0) {
 196                 DRM_ERROR("%s initialization failed "
 197                                 "ctl %08x head %08x tail %08x start %08x\n",
 198                                 ring->name,
 199                                 I915_READ(ring->regs.ctl),
 200                                 I915_READ(ring->regs.head),
 201                                 I915_READ(ring->regs.tail),
 202                                 I915_READ(ring->regs.start));
 203                 return -EIO;
 204         }
 205
 206         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 207                 i915_kernel_lost_context(dev);
 208         else {
 209                 ring->head = ring->get_head(dev, ring);
 210                 ring->tail = ring->get_tail(dev, ring);
 211                 ring->space = ring->head - (ring->tail + 8);
 212                 if (ring->space < 0)
 213                         ring->space += ring->size;
 214         }
 215         return 0;
 216 }
 217
 218 static int init_render_ring(struct drm_device *dev,
 219                 struct intel_ring_buffer *ring)
 220 {
 221         drm_i915_private_t *dev_priv = dev->dev_private;
 222         int ret = init_ring_common(dev, ring);
 223         int mode;
 224
 225         if (IS_I9XX(dev) && !IS_GEN3(dev)) {
 226                 mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
 227                 if (IS_GEN6(dev))
 228                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
 229                 I915_WRITE(MI_MODE, mode);
 230         }
 231         return ret;
 232 }
 233
 234 #define PIPE_CONTROL_FLUSH(addr)                                        \
 235 do {                                                                    \
 236         OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
 237                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
 238         OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
 239         OUT_RING(0);                                                    \
 240         OUT_RING(0);                                                    \
 241 } while (0)
 242
 243 /**
 244  * Creates a new sequence number, emitting a write of it to the status page
 245  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 246  *
 247  * Must be called with struct_lock held.
 248  *
 249  * Returned sequence numbers are nonzero on success.
 250  */
 251 static u32
 252 render_ring_add_request(struct drm_device *dev,
 253                 struct intel_ring_buffer *ring,
 254                 struct drm_file *file_priv,
 255                 u32 flush_domains)
 256 {
 257         drm_i915_private_t *dev_priv = dev->dev_private;
 258         u32 seqno;
 259
 260         seqno = i915_gem_get_seqno(dev);
 261
 262         if (IS_GEN6(dev)) {
 263                 BEGIN_LP_RING(6);
 264                 OUT_RING(GFX_OP_PIPE_CONTROL | 3);
 265                 OUT_RING(PIPE_CONTROL_QW_WRITE |
 266                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
 267                          PIPE_CONTROL_NOTIFY);
 268                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 269                 OUT_RING(seqno);
 270                 OUT_RING(0);
 271                 OUT_RING(0);
 272                 ADVANCE_LP_RING();
 273         } else if (HAS_PIPE_CONTROL(dev)) {
 274                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
 275
 276                 /*
 277                  * Workaround qword write incoherence by flushing the
 278                  * PIPE_NOTIFY buffers out to memory before requesting
 279                  * an interrupt.
 280                  */
 281                 BEGIN_LP_RING(32);
 282                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 283                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
 284                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 285                 OUT_RING(seqno);
 286                 OUT_RING(0);
 287                 PIPE_CONTROL_FLUSH(scratch_addr);
 288                 scratch_addr += 128; /* write to separate cachelines */
 289                 PIPE_CONTROL_FLUSH(scratch_addr);
 290                 scratch_addr += 128;
 291                 PIPE_CONTROL_FLUSH(scratch_addr);
 292                 scratch_addr += 128;
 293                 PIPE_CONTROL_FLUSH(scratch_addr);
 294                 scratch_addr += 128;
 295                 PIPE_CONTROL_FLUSH(scratch_addr);
 296                 scratch_addr += 128;
 297                 PIPE_CONTROL_FLUSH(scratch_addr);
 298                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 299                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
 300                          PIPE_CONTROL_NOTIFY);
 301                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 302                 OUT_RING(seqno);
 303                 OUT_RING(0);
 304                 ADVANCE_LP_RING();
 305         } else {
 306                 BEGIN_LP_RING(4);
 307                 OUT_RING(MI_STORE_DWORD_INDEX);
 308                 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 309                 OUT_RING(seqno);
 310
 311                 OUT_RING(MI_USER_INTERRUPT);
 312                 ADVANCE_LP_RING();
 313         }
 314         return seqno;
 315 }
 316
 317 static u32
 318 render_ring_get_gem_seqno(struct drm_device *dev,
 319                 struct intel_ring_buffer *ring)
 320 {
 321         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 322         if (HAS_PIPE_CONTROL(dev))
 323                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
 324         else
 325                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 326 }
 327
 328 static void
 329 render_ring_get_user_irq(struct drm_device *dev,
 330                 struct intel_ring_buffer *ring)
 331 {
 332         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 333         unsigned long irqflags;
 334
 335         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 336         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
 337                 if (HAS_PCH_SPLIT(dev))
 338                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 339                 else
 340                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 341         }
 342         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 343 }
 344
 345 static void
 346 render_ring_put_user_irq(struct drm_device *dev,
 347                 struct intel_ring_buffer *ring)
 348 {
 349         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 350         unsigned long irqflags;
 351
 352         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 353         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
 354         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
 355                 if (HAS_PCH_SPLIT(dev))
 356                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 357                 else
 358                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 359         }
 360         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 361 }
 362
 363 static void render_setup_status_page(struct drm_device *dev,
 364         struct  intel_ring_buffer *ring)
 365 {
 366         drm_i915_private_t *dev_priv = dev->dev_private;
 367         if (IS_GEN6(dev)) {
 368                 I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr);
 369                 I915_READ(HWS_PGA_GEN6); /* posting read */
 370         } else {
 371                 I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
 372                 I915_READ(HWS_PGA); /* posting read */
 373         }
 374
 375 }
 376
 377 void
 378 bsd_ring_flush(struct drm_device *dev,
 379                 struct intel_ring_buffer *ring,
 380                 u32     invalidate_domains,
 381                 u32     flush_domains)
 382 {
 383         intel_ring_begin(dev, ring, 2);
 384         intel_ring_emit(dev, ring, MI_FLUSH);
 385         intel_ring_emit(dev, ring, MI_NOOP);
 386         intel_ring_advance(dev, ring);
 387 }
 388
 389 static inline unsigned int bsd_ring_get_head(struct drm_device *dev,
 390                 struct intel_ring_buffer *ring)
 391 {
 392         drm_i915_private_t *dev_priv = dev->dev_private;
 393         return I915_READ(BSD_RING_HEAD) & HEAD_ADDR;
 394 }
 395
 396 static inline unsigned int bsd_ring_get_tail(struct drm_device *dev,
 397                 struct intel_ring_buffer *ring)
 398 {
 399         drm_i915_private_t *dev_priv = dev->dev_private;
 400         return I915_READ(BSD_RING_TAIL) & TAIL_ADDR;
 401 }
 402
 403 static inline unsigned int bsd_ring_get_active_head(struct drm_device *dev,
 404                 struct intel_ring_buffer *ring)
 405 {
 406         drm_i915_private_t *dev_priv = dev->dev_private;
 407         return I915_READ(BSD_RING_ACTHD);
 408 }
 409
 410 static inline void bsd_ring_advance_ring(struct drm_device *dev,
 411                 struct intel_ring_buffer *ring)
 412 {
 413         drm_i915_private_t *dev_priv = dev->dev_private;
 414         I915_WRITE(BSD_RING_TAIL, ring->tail);
 415 }
 416
 417 static int init_bsd_ring(struct drm_device *dev,
 418                 struct intel_ring_buffer *ring)
 419 {
 420         return init_ring_common(dev, ring);
 421 }
 422
 423 static u32
 424 bsd_ring_add_request(struct drm_device *dev,
 425                 struct intel_ring_buffer *ring,
 426                 struct drm_file *file_priv,
 427                 u32 flush_domains)
 428 {
 429         u32 seqno;
 430
 431         seqno = i915_gem_get_seqno(dev);
 432
 433         intel_ring_begin(dev, ring, 4);
 434         intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
 435         intel_ring_emit(dev, ring,
 436                         I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 437         intel_ring_emit(dev, ring, seqno);
 438         intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
 439         intel_ring_advance(dev, ring);
 440
 441         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
 442
 443         return seqno;
 444 }
 445
 446 static void bsd_setup_status_page(struct drm_device *dev,
 447                 struct  intel_ring_buffer *ring)
 448 {
 449         drm_i915_private_t *dev_priv = dev->dev_private;
 450         I915_WRITE(BSD_HWS_PGA, ring->status_page.gfx_addr);
 451         I915_READ(BSD_HWS_PGA);
 452 }
 453
 454 static void
 455 bsd_ring_get_user_irq(struct drm_device *dev,
 456                 struct intel_ring_buffer *ring)
 457 {
 458         /* do nothing */
 459 }
 460 static void
 461 bsd_ring_put_user_irq(struct drm_device *dev,
 462                 struct intel_ring_buffer *ring)
 463 {
 464         /* do nothing */
 465 }
 466
 467 static u32
 468 bsd_ring_get_gem_seqno(struct drm_device *dev,
 469                 struct intel_ring_buffer *ring)
 470 {
 471         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 472 }
 473
 474 static int
 475 bsd_ring_dispatch_gem_execbuffer(struct drm_device *dev,
 476                 struct intel_ring_buffer *ring,
 477                 struct drm_i915_gem_execbuffer2 *exec,
 478                 struct drm_clip_rect *cliprects,
 479                 uint64_t exec_offset)
 480 {
 481         uint32_t exec_start;
 482         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 483         intel_ring_begin(dev, ring, 2);
 484         intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START |
 485                         (2 << 6) | MI_BATCH_NON_SECURE_I965);
 486         intel_ring_emit(dev, ring, exec_start);
 487         intel_ring_advance(dev, ring);
 488         return 0;
 489 }
 490
 491
 492 static int
 493 render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
 494                 struct intel_ring_buffer *ring,
 495                 struct drm_i915_gem_execbuffer2 *exec,
 496                 struct drm_clip_rect *cliprects,
 497                 uint64_t exec_offset)
 498 {
 499         drm_i915_private_t *dev_priv = dev->dev_private;
 500         int nbox = exec->num_cliprects;
 501         int i = 0, count;
 502         uint32_t exec_start, exec_len;
 503         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 504         exec_len = (uint32_t) exec->batch_len;
 505
 506         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
 507
 508         count = nbox ? nbox : 1;
 509
 510         for (i = 0; i < count; i++) {
 511                 if (i < nbox) {
 512                         int ret = i915_emit_box(dev, cliprects, i,
 513                                                 exec->DR1, exec->DR4);
 514                         if (ret)
 515                                 return ret;
 516                 }
 517
 518                 if (IS_I830(dev) || IS_845G(dev)) {
 519                         intel_ring_begin(dev, ring, 4);
 520                         intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
 521                         intel_ring_emit(dev, ring,
 522                                         exec_start | MI_BATCH_NON_SECURE);
 523                         intel_ring_emit(dev, ring, exec_start + exec_len - 4);
 524                         intel_ring_emit(dev, ring, 0);
 525                 } else {
 526                         intel_ring_begin(dev, ring, 4);
 527                         if (IS_I965G(dev)) {
 528                                 intel_ring_emit(dev, ring,
 529                                                 MI_BATCH_BUFFER_START | (2 << 6)
 530                                                 | MI_BATCH_NON_SECURE_I965);
 531                                 intel_ring_emit(dev, ring, exec_start);
 532                         } else {
 533                                 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
 534                                                 | (2 << 6));
 535                                 intel_ring_emit(dev, ring, exec_start |
 536                                                 MI_BATCH_NON_SECURE);
 537                         }
 538                 }
 539                 intel_ring_advance(dev, ring);
 540         }
 541
 542         if (IS_G4X(dev) || IS_IRONLAKE(dev)) {
 543                 intel_ring_begin(dev, ring, 2);
 544                 intel_ring_emit(dev, ring, MI_FLUSH |
 545                                 MI_NO_WRITE_FLUSH |
 546                                 MI_INVALIDATE_ISP );
 547                 intel_ring_emit(dev, ring, MI_NOOP);
 548                 intel_ring_advance(dev, ring);
 549         }
 550         /* XXX breadcrumb */
 551
 552         return 0;
 553 }
 554
 555 static void cleanup_status_page(struct drm_device *dev,
 556                 struct intel_ring_buffer *ring)
 557 {
 558         drm_i915_private_t *dev_priv = dev->dev_private;
 559         struct drm_gem_object *obj;
 560         struct drm_i915_gem_object *obj_priv;
 561
 562         obj = ring->status_page.obj;
 563         if (obj == NULL)
 564                 return;
 565         obj_priv = to_intel_bo(obj);
 566
 567         kunmap(obj_priv->pages[0]);
 568         i915_gem_object_unpin(obj);
 569         drm_gem_object_unreference(obj);
 570         ring->status_page.obj = NULL;
 571
 572         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 573 }
 574
 575 static int init_status_page(struct drm_device *dev,
 576                 struct intel_ring_buffer *ring)
 577 {
 578         drm_i915_private_t *dev_priv = dev->dev_private;
 579         struct drm_gem_object *obj;
 580         struct drm_i915_gem_object *obj_priv;
 581         int ret;
 582
 583         obj = i915_gem_alloc_object(dev, 4096);
 584         if (obj == NULL) {
 585                 DRM_ERROR("Failed to allocate status page\n");
 586                 ret = -ENOMEM;
 587                 goto err;
 588         }
 589         obj_priv = to_intel_bo(obj);
 590         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
 591
 592         ret = i915_gem_object_pin(obj, 4096);
 593         if (ret != 0) {
 594                 goto err_unref;
 595         }
 596
 597         ring->status_page.gfx_addr = obj_priv->gtt_offset;
 598         ring->status_page.page_addr = kmap(obj_priv->pages[0]);
 599         if (ring->status_page.page_addr == NULL) {
 600                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 601                 goto err_unpin;
 602         }
 603         ring->status_page.obj = obj;
 604         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 605
 606         ring->setup_status_page(dev, ring);
 607         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 608                         ring->name, ring->status_page.gfx_addr);
 609
 610         return 0;
 611
 612 err_unpin:
 613         i915_gem_object_unpin(obj);
 614 err_unref:
 615         drm_gem_object_unreference(obj);
 616 err:
 617         return ret;
 618 }
 619
 620
 621 int intel_init_ring_buffer(struct drm_device *dev,
 622                 struct intel_ring_buffer *ring)
 623 {
 624         struct drm_i915_gem_object *obj_priv;
 625         struct drm_gem_object *obj;
 626         int ret;
 627
 628         ring->dev = dev;
 629
 630         if (I915_NEED_GFX_HWS(dev)) {
 631                 ret = init_status_page(dev, ring);
 632                 if (ret)
 633                         return ret;
 634         }
 635
 636         obj = i915_gem_alloc_object(dev, ring->size);
 637         if (obj == NULL) {
 638                 DRM_ERROR("Failed to allocate ringbuffer\n");
 639                 ret = -ENOMEM;
 640                 goto err_hws;
 641         }
 642
 643         ring->gem_object = obj;
 644
 645         ret = i915_gem_object_pin(obj, ring->alignment);
 646         if (ret)
 647                 goto err_unref;
 648
 649         obj_priv = to_intel_bo(obj);
 650         ring->map.size = ring->size;
 651         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
 652         ring->map.type = 0;
 653         ring->map.flags = 0;
 654         ring->map.mtrr = 0;
 655
 656         drm_core_ioremap_wc(&ring->map, dev);
 657         if (ring->map.handle == NULL) {
 658                 DRM_ERROR("Failed to map ringbuffer.\n");
 659                 ret = -EINVAL;
 660                 goto err_unpin;
 661         }
 662
 663         ring->virtual_start = ring->map.handle;
 664         ret = ring->init(dev, ring);
 665         if (ret)
 666                 goto err_unmap;
 667
 668         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 669                 i915_kernel_lost_context(dev);
 670         else {
 671                 ring->head = ring->get_head(dev, ring);
 672                 ring->tail = ring->get_tail(dev, ring);
 673                 ring->space = ring->head - (ring->tail + 8);
 674                 if (ring->space < 0)
 675                         ring->space += ring->size;
 676         }
 677         INIT_LIST_HEAD(&ring->active_list);
 678         INIT_LIST_HEAD(&ring->request_list);
 679         return ret;
 680
 681 err_unmap:
 682         drm_core_ioremapfree(&ring->map, dev);
 683 err_unpin:
 684         i915_gem_object_unpin(obj);
 685 err_unref:
 686         drm_gem_object_unreference(obj);
 687         ring->gem_object = NULL;
 688 err_hws:
 689         cleanup_status_page(dev, ring);
 690         return ret;
 691 }
 692
 693 void intel_cleanup_ring_buffer(struct drm_device *dev,
 694                 struct intel_ring_buffer *ring)
 695 {
 696         if (ring->gem_object == NULL)
 697                 return;
 698
 699         drm_core_ioremapfree(&ring->map, dev);
 700
 701         i915_gem_object_unpin(ring->gem_object);
 702         drm_gem_object_unreference(ring->gem_object);
 703         ring->gem_object = NULL;
 704         cleanup_status_page(dev, ring);
 705 }
 706
 707 int intel_wrap_ring_buffer(struct drm_device *dev,
 708                 struct intel_ring_buffer *ring)
 709 {
 710         unsigned int *virt;
 711         int rem;
 712         rem = ring->size - ring->tail;
 713
 714         if (ring->space < rem) {
 715                 int ret = intel_wait_ring_buffer(dev, ring, rem);
 716                 if (ret)
 717                         return ret;
 718         }
 719
 720         virt = (unsigned int *)(ring->virtual_start + ring->tail);
 721         rem /= 8;
 722         while (rem--) {
 723                 *virt++ = MI_NOOP;
 724                 *virt++ = MI_NOOP;
 725         }
 726
 727         ring->tail = 0;
 728         ring->space = ring->head - 8;
 729
 730         return 0;
 731 }
 732
 733 int intel_wait_ring_buffer(struct drm_device *dev,
 734                 struct intel_ring_buffer *ring, int n)
 735 {
 736         unsigned long end;
 737
 738         trace_i915_ring_wait_begin (dev);
 739         end = jiffies + 3 * HZ;
 740         do {
 741                 ring->head = ring->get_head(dev, ring);
 742                 ring->space = ring->head - (ring->tail + 8);
 743                 if (ring->space < 0)
 744                         ring->space += ring->size;
 745                 if (ring->space >= n) {
 746                         trace_i915_ring_wait_end (dev);
 747                         return 0;
 748                 }
 749
 750                 if (dev->primary->master) {
 751                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 752                         if (master_priv->sarea_priv)
 753                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 754                 }
 755
 756                 yield();
 757         } while (!time_after(jiffies, end));
 758         trace_i915_ring_wait_end (dev);
 759         return -EBUSY;
 760 }
 761
 762 void intel_ring_begin(struct drm_device *dev,
 763                 struct intel_ring_buffer *ring, int num_dwords)
 764 {
 765         int n = 4*num_dwords;
 766         if (unlikely(ring->tail + n > ring->size))
 767                 intel_wrap_ring_buffer(dev, ring);
 768         if (unlikely(ring->space < n))
 769                 intel_wait_ring_buffer(dev, ring, n);
 770
 771         ring->space -= n;
 772 }
 773
 774 void intel_ring_advance(struct drm_device *dev,
 775                 struct intel_ring_buffer *ring)
 776 {
 777         ring->tail &= ring->size - 1;
 778         ring->advance_ring(dev, ring);
 779 }
 780
 781 void intel_fill_struct(struct drm_device *dev,
 782                 struct intel_ring_buffer *ring,
 783                 void *data,
 784                 unsigned int len)
 785 {
 786         unsigned int *virt = ring->virtual_start + ring->tail;
 787         BUG_ON((len&~(4-1)) != 0);
 788         intel_ring_begin(dev, ring, len/4);
 789         memcpy(virt, data, len);
 790         ring->tail += len;
 791         ring->tail &= ring->size - 1;
 792         ring->space -= len;
 793         intel_ring_advance(dev, ring);
 794 }
 795
 796 struct intel_ring_buffer render_ring = {
 797         .name                   = "render ring",
 798         .regs                   = {
 799                 .ctl = PRB0_CTL,
 800                 .head = PRB0_HEAD,
 801                 .tail = PRB0_TAIL,
 802                 .start = PRB0_START
 803         },
 804         .ring_flag              = I915_EXEC_RENDER,
 805         .size                   = 32 * PAGE_SIZE,
 806         .alignment              = PAGE_SIZE,
 807         .virtual_start          = NULL,
 808         .dev                    = NULL,
 809         .gem_object             = NULL,
 810         .head                   = 0,
 811         .tail                   = 0,
 812         .space                  = 0,
 813         .user_irq_refcount      = 0,
 814         .irq_gem_seqno          = 0,
 815         .waiting_gem_seqno      = 0,
 816         .setup_status_page      = render_setup_status_page,
 817         .init                   = init_render_ring,
 818         .get_head               = render_ring_get_head,
 819         .get_tail               = render_ring_get_tail,
 820         .get_active_head        = render_ring_get_active_head,
 821         .advance_ring           = render_ring_advance_ring,
 822         .flush                  = render_ring_flush,
 823         .add_request            = render_ring_add_request,
 824         .get_gem_seqno          = render_ring_get_gem_seqno,
 825         .user_irq_get           = render_ring_get_user_irq,
 826         .user_irq_put           = render_ring_put_user_irq,
 827         .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
 828         .status_page            = {NULL, 0, NULL},
 829         .map                    = {0,}
 830 };
 831
 832 /* ring buffer for bit-stream decoder */
 833
 834 struct intel_ring_buffer bsd_ring = {
 835         .name                   = "bsd ring",
 836         .regs                   = {
 837                 .ctl = BSD_RING_CTL,
 838                 .head = BSD_RING_HEAD,
 839                 .tail = BSD_RING_TAIL,
 840                 .start = BSD_RING_START
 841         },
 842         .ring_flag              = I915_EXEC_BSD,
 843         .size                   = 32 * PAGE_SIZE,
 844         .alignment              = PAGE_SIZE,
 845         .virtual_start          = NULL,
 846         .dev                    = NULL,
 847         .gem_object             = NULL,
 848         .head                   = 0,
 849         .tail                   = 0,
 850         .space                  = 0,
 851         .user_irq_refcount      = 0,
 852         .irq_gem_seqno          = 0,
 853         .waiting_gem_seqno      = 0,
 854         .setup_status_page      = bsd_setup_status_page,
 855         .init                   = init_bsd_ring,
 856         .get_head               = bsd_ring_get_head,
 857         .get_tail               = bsd_ring_get_tail,
 858         .get_active_head        = bsd_ring_get_active_head,
 859         .advance_ring           = bsd_ring_advance_ring,
 860         .flush                  = bsd_ring_flush,
 861         .add_request            = bsd_ring_add_request,
 862         .get_gem_seqno          = bsd_ring_get_gem_seqno,
 863         .user_irq_get           = bsd_ring_get_user_irq,
 864         .user_irq_put           = bsd_ring_put_user_irq,
 865         .dispatch_gem_execbuffer = bsd_ring_dispatch_gem_execbuffer,
 866         .status_page            = {NULL, 0, NULL},
 867         .map                    = {0,}
 868 };