drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  40
  41 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  42                                                   bool pipelined);
  43 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  44 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  45 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  46                                              int write);
  47 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  48                                                      uint64_t offset,
  49                                                      uint64_t size);
  50 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  51 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  52                                           bool interruptible);
  53 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  54                                            unsigned alignment);
  55 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  56 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  57                                 struct drm_i915_gem_pwrite *args,
  58                                 struct drm_file *file_priv);
  59 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  60
  61 static LIST_HEAD(shrink_list);
  62 static DEFINE_SPINLOCK(shrink_list_lock);
  63
  64 int
  65 i915_gem_check_is_wedged(struct drm_device *dev)
  66 {
  67         struct drm_i915_private *dev_priv = dev->dev_private;
  68         struct completion *x = &dev_priv->error_completion;
  69         unsigned long flags;
  70         int ret;
  71
  72         if (!atomic_read(&dev_priv->mm.wedged))
  73                 return 0;
  74
  75         ret = wait_for_completion_interruptible(x);
  76         if (ret)
  77                 return ret;
  78
  79         /* Success, we reset the GPU! */
  80         if (!atomic_read(&dev_priv->mm.wedged))
  81                 return 0;
  82
  83         /* GPU is hung, bump the completion count to account for
  84          * the token we just consumed so that we never hit zero and
  85          * end up waiting upon a subsequent completion event that
  86          * will never happen.
  87          */
  88         spin_lock_irqsave(&x->wait.lock, flags);
  89         x->done++;
  90         spin_unlock_irqrestore(&x->wait.lock, flags);
  91         return -EIO;
  92 }
  93
  94 static int i915_mutex_lock_interruptible(struct drm_device *dev)
  95 {
  96         struct drm_i915_private *dev_priv = dev->dev_private;
  97         int ret;
  98
  99         ret = i915_gem_check_is_wedged(dev);
 100         if (ret)
 101                 return ret;
 102
 103         ret = mutex_lock_interruptible(&dev->struct_mutex);
 104         if (ret)
 105                 return ret;
 106
 107         if (atomic_read(&dev_priv->mm.wedged)) {
 108                 mutex_unlock(&dev->struct_mutex);
 109                 return -EAGAIN;
 110         }
 111
 112         return 0;
 113 }
 114
 115 static inline bool
 116 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 117 {
 118         return obj_priv->gtt_space &&
 119                 !obj_priv->active &&
 120                 obj_priv->pin_count == 0;
 121 }
 122
 123 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
 124                      unsigned long end)
 125 {
 126         drm_i915_private_t *dev_priv = dev->dev_private;
 127
 128         if (start >= end ||
 129             (start & (PAGE_SIZE - 1)) != 0 ||
 130             (end & (PAGE_SIZE - 1)) != 0) {
 131                 return -EINVAL;
 132         }
 133
 134         drm_mm_init(&dev_priv->mm.gtt_space, start,
 135                     end - start);
 136
 137         dev->gtt_total = (uint32_t) (end - start);
 138
 139         return 0;
 140 }
 141
 142 int
 143 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 144                     struct drm_file *file_priv)
 145 {
 146         struct drm_i915_gem_init *args = data;
 147         int ret;
 148
 149         mutex_lock(&dev->struct_mutex);
 150         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
 151         mutex_unlock(&dev->struct_mutex);
 152
 153         return ret;
 154 }
 155
 156 int
 157 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 158                             struct drm_file *file_priv)
 159 {
 160         struct drm_i915_gem_get_aperture *args = data;
 161
 162         if (!(dev->driver->driver_features & DRIVER_GEM))
 163                 return -ENODEV;
 164
 165         args->aper_size = dev->gtt_total;
 166         args->aper_available_size = (args->aper_size -
 167                                      atomic_read(&dev->pin_memory));
 168
 169         return 0;
 170 }
 171
 172
 173 /**
 174  * Creates a new mm object and returns a handle to it.
 175  */
 176 int
 177 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 178                       struct drm_file *file_priv)
 179 {
 180         struct drm_i915_gem_create *args = data;
 181         struct drm_gem_object *obj;
 182         int ret;
 183         u32 handle;
 184
 185         args->size = roundup(args->size, PAGE_SIZE);
 186
 187         /* Allocate the new object */
 188         obj = i915_gem_alloc_object(dev, args->size);
 189         if (obj == NULL)
 190                 return -ENOMEM;
 191
 192         ret = drm_gem_handle_create(file_priv, obj, &handle);
 193         if (ret) {
 194                 drm_gem_object_unreference_unlocked(obj);
 195                 return ret;
 196         }
 197
 198         /* Sink the floating reference from kref_init(handlecount) */
 199         drm_gem_object_handle_unreference_unlocked(obj);
 200
 201         args->handle = handle;
 202         return 0;
 203 }
 204
 205 static inline int
 206 fast_shmem_read(struct page **pages,
 207                 loff_t page_base, int page_offset,
 208                 char __user *data,
 209                 int length)
 210 {
 211         char __iomem *vaddr;
 212         int unwritten;
 213
 214         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 215         if (vaddr == NULL)
 216                 return -ENOMEM;
 217         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 218         kunmap_atomic(vaddr, KM_USER0);
 219
 220         if (unwritten)
 221                 return -EFAULT;
 222
 223         return 0;
 224 }
 225
 226 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 227 {
 228         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 229         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 230
 231         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 232                 obj_priv->tiling_mode != I915_TILING_NONE;
 233 }
 234
 235 static inline void
 236 slow_shmem_copy(struct page *dst_page,
 237                 int dst_offset,
 238                 struct page *src_page,
 239                 int src_offset,
 240                 int length)
 241 {
 242         char *dst_vaddr, *src_vaddr;
 243
 244         dst_vaddr = kmap(dst_page);
 245         src_vaddr = kmap(src_page);
 246
 247         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 248
 249         kunmap(src_page);
 250         kunmap(dst_page);
 251 }
 252
 253 static inline void
 254 slow_shmem_bit17_copy(struct page *gpu_page,
 255                       int gpu_offset,
 256                       struct page *cpu_page,
 257                       int cpu_offset,
 258                       int length,
 259                       int is_read)
 260 {
 261         char *gpu_vaddr, *cpu_vaddr;
 262
 263         /* Use the unswizzled path if this page isn't affected. */
 264         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 265                 if (is_read)
 266                         return slow_shmem_copy(cpu_page, cpu_offset,
 267                                                gpu_page, gpu_offset, length);
 268                 else
 269                         return slow_shmem_copy(gpu_page, gpu_offset,
 270                                                cpu_page, cpu_offset, length);
 271         }
 272
 273         gpu_vaddr = kmap(gpu_page);
 274         cpu_vaddr = kmap(cpu_page);
 275
 276         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 277          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 278          */
 279         while (length > 0) {
 280                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 281                 int this_length = min(cacheline_end - gpu_offset, length);
 282                 int swizzled_gpu_offset = gpu_offset ^ 64;
 283
 284                 if (is_read) {
 285                         memcpy(cpu_vaddr + cpu_offset,
 286                                gpu_vaddr + swizzled_gpu_offset,
 287                                this_length);
 288                 } else {
 289                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 290                                cpu_vaddr + cpu_offset,
 291                                this_length);
 292                 }
 293                 cpu_offset += this_length;
 294                 gpu_offset += this_length;
 295                 length -= this_length;
 296         }
 297
 298         kunmap(cpu_page);
 299         kunmap(gpu_page);
 300 }
 301
 302 /**
 303  * This is the fast shmem pread path, which attempts to copy_from_user directly
 304  * from the backing pages of the object to the user's address space.  On a
 305  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 306  */
 307 static int
 308 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 309                           struct drm_i915_gem_pread *args,
 310                           struct drm_file *file_priv)
 311 {
 312         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 313         ssize_t remain;
 314         loff_t offset, page_base;
 315         char __user *user_data;
 316         int page_offset, page_length;
 317         int ret;
 318
 319         user_data = (char __user *) (uintptr_t) args->data_ptr;
 320         remain = args->size;
 321
 322         ret = i915_mutex_lock_interruptible(dev);
 323         if (ret)
 324                 return ret;
 325
 326         ret = i915_gem_object_get_pages(obj, 0);
 327         if (ret != 0)
 328                 goto fail_unlock;
 329
 330         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 331                                                         args->size);
 332         if (ret != 0)
 333                 goto fail_put_pages;
 334
 335         obj_priv = to_intel_bo(obj);
 336         offset = args->offset;
 337
 338         while (remain > 0) {
 339                 /* Operation in this page
 340                  *
 341                  * page_base = page offset within aperture
 342                  * page_offset = offset within page
 343                  * page_length = bytes to copy for this page
 344                  */
 345                 page_base = (offset & ~(PAGE_SIZE-1));
 346                 page_offset = offset & (PAGE_SIZE-1);
 347                 page_length = remain;
 348                 if ((page_offset + remain) > PAGE_SIZE)
 349                         page_length = PAGE_SIZE - page_offset;
 350
 351                 ret = fast_shmem_read(obj_priv->pages,
 352                                       page_base, page_offset,
 353                                       user_data, page_length);
 354                 if (ret)
 355                         goto fail_put_pages;
 356
 357                 remain -= page_length;
 358                 user_data += page_length;
 359                 offset += page_length;
 360         }
 361
 362 fail_put_pages:
 363         i915_gem_object_put_pages(obj);
 364 fail_unlock:
 365         mutex_unlock(&dev->struct_mutex);
 366
 367         return ret;
 368 }
 369
 370 static int
 371 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 372 {
 373         int ret;
 374
 375         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 376
 377         /* If we've insufficient memory to map in the pages, attempt
 378          * to make some space by throwing out some old buffers.
 379          */
 380         if (ret == -ENOMEM) {
 381                 struct drm_device *dev = obj->dev;
 382
 383                 ret = i915_gem_evict_something(dev, obj->size,
 384                                                i915_gem_get_gtt_alignment(obj));
 385                 if (ret)
 386                         return ret;
 387
 388                 ret = i915_gem_object_get_pages(obj, 0);
 389         }
 390
 391         return ret;
 392 }
 393
 394 /**
 395  * This is the fallback shmem pread path, which allocates temporary storage
 396  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 397  * can copy out of the object's backing pages while holding the struct mutex
 398  * and not take page faults.
 399  */
 400 static int
 401 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 402                           struct drm_i915_gem_pread *args,
 403                           struct drm_file *file_priv)
 404 {
 405         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 406         struct mm_struct *mm = current->mm;
 407         struct page **user_pages;
 408         ssize_t remain;
 409         loff_t offset, pinned_pages, i;
 410         loff_t first_data_page, last_data_page, num_pages;
 411         int shmem_page_index, shmem_page_offset;
 412         int data_page_index,  data_page_offset;
 413         int page_length;
 414         int ret;
 415         uint64_t data_ptr = args->data_ptr;
 416         int do_bit17_swizzling;
 417
 418         remain = args->size;
 419
 420         /* Pin the user pages containing the data.  We can't fault while
 421          * holding the struct mutex, yet we want to hold it while
 422          * dereferencing the user data.
 423          */
 424         first_data_page = data_ptr / PAGE_SIZE;
 425         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 426         num_pages = last_data_page - first_data_page + 1;
 427
 428         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 429         if (user_pages == NULL)
 430                 return -ENOMEM;
 431
 432         down_read(&mm->mmap_sem);
 433         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 434                                       num_pages, 1, 0, user_pages, NULL);
 435         up_read(&mm->mmap_sem);
 436         if (pinned_pages < num_pages) {
 437                 ret = -EFAULT;
 438                 goto fail_put_user_pages;
 439         }
 440
 441         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 442
 443         ret = i915_mutex_lock_interruptible(dev);
 444         if (ret)
 445                 goto fail_put_user_pages;
 446
 447         ret = i915_gem_object_get_pages_or_evict(obj);
 448         if (ret)
 449                 goto fail_unlock;
 450
 451         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 452                                                         args->size);
 453         if (ret != 0)
 454                 goto fail_put_pages;
 455
 456         obj_priv = to_intel_bo(obj);
 457         offset = args->offset;
 458
 459         while (remain > 0) {
 460                 /* Operation in this page
 461                  *
 462                  * shmem_page_index = page number within shmem file
 463                  * shmem_page_offset = offset within page in shmem file
 464                  * data_page_index = page number in get_user_pages return
 465                  * data_page_offset = offset with data_page_index page.
 466                  * page_length = bytes to copy for this page
 467                  */
 468                 shmem_page_index = offset / PAGE_SIZE;
 469                 shmem_page_offset = offset & ~PAGE_MASK;
 470                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 471                 data_page_offset = data_ptr & ~PAGE_MASK;
 472
 473                 page_length = remain;
 474                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 475                         page_length = PAGE_SIZE - shmem_page_offset;
 476                 if ((data_page_offset + page_length) > PAGE_SIZE)
 477                         page_length = PAGE_SIZE - data_page_offset;
 478
 479                 if (do_bit17_swizzling) {
 480                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 481                                               shmem_page_offset,
 482                                               user_pages[data_page_index],
 483                                               data_page_offset,
 484                                               page_length,
 485                                               1);
 486                 } else {
 487                         slow_shmem_copy(user_pages[data_page_index],
 488                                         data_page_offset,
 489                                         obj_priv->pages[shmem_page_index],
 490                                         shmem_page_offset,
 491                                         page_length);
 492                 }
 493
 494                 remain -= page_length;
 495                 data_ptr += page_length;
 496                 offset += page_length;
 497         }
 498
 499 fail_put_pages:
 500         i915_gem_object_put_pages(obj);
 501 fail_unlock:
 502         mutex_unlock(&dev->struct_mutex);
 503 fail_put_user_pages:
 504         for (i = 0; i < pinned_pages; i++) {
 505                 SetPageDirty(user_pages[i]);
 506                 page_cache_release(user_pages[i]);
 507         }
 508         drm_free_large(user_pages);
 509
 510         return ret;
 511 }
 512
 513 /**
 514  * Reads data from the object referenced by handle.
 515  *
 516  * On error, the contents of *data are undefined.
 517  */
 518 int
 519 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 520                      struct drm_file *file_priv)
 521 {
 522         struct drm_i915_gem_pread *args = data;
 523         struct drm_gem_object *obj;
 524         struct drm_i915_gem_object *obj_priv;
 525         int ret;
 526
 527         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 528         if (obj == NULL)
 529                 return -ENOENT;
 530         obj_priv = to_intel_bo(obj);
 531
 532         /* Bounds check source.
 533          *
 534          * XXX: This could use review for overflow issues...
 535          */
 536         if (args->offset > obj->size || args->size > obj->size ||
 537             args->offset + args->size > obj->size) {
 538                 drm_gem_object_unreference_unlocked(obj);
 539                 return -EINVAL;
 540         }
 541
 542         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 543                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 544         } else {
 545                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 546                 if (ret != 0)
 547                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 548                                                         file_priv);
 549         }
 550
 551         drm_gem_object_unreference_unlocked(obj);
 552
 553         return ret;
 554 }
 555
 556 /* This is the fast write path which cannot handle
 557  * page faults in the source data
 558  */
 559
 560 static inline int
 561 fast_user_write(struct io_mapping *mapping,
 562                 loff_t page_base, int page_offset,
 563                 char __user *user_data,
 564                 int length)
 565 {
 566         char *vaddr_atomic;
 567         unsigned long unwritten;
 568
 569         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
 570         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 571                                                       user_data, length);
 572         io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
 573         if (unwritten)
 574                 return -EFAULT;
 575         return 0;
 576 }
 577
 578 /* Here's the write path which can sleep for
 579  * page faults
 580  */
 581
 582 static inline void
 583 slow_kernel_write(struct io_mapping *mapping,
 584                   loff_t gtt_base, int gtt_offset,
 585                   struct page *user_page, int user_offset,
 586                   int length)
 587 {
 588         char __iomem *dst_vaddr;
 589         char *src_vaddr;
 590
 591         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 592         src_vaddr = kmap(user_page);
 593
 594         memcpy_toio(dst_vaddr + gtt_offset,
 595                     src_vaddr + user_offset,
 596                     length);
 597
 598         kunmap(user_page);
 599         io_mapping_unmap(dst_vaddr);
 600 }
 601
 602 static inline int
 603 fast_shmem_write(struct page **pages,
 604                  loff_t page_base, int page_offset,
 605                  char __user *data,
 606                  int length)
 607 {
 608         char __iomem *vaddr;
 609         unsigned long unwritten;
 610
 611         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 612         if (vaddr == NULL)
 613                 return -ENOMEM;
 614         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 615         kunmap_atomic(vaddr, KM_USER0);
 616
 617         if (unwritten)
 618                 return -EFAULT;
 619         return 0;
 620 }
 621
 622 /**
 623  * This is the fast pwrite path, where we copy the data directly from the
 624  * user into the GTT, uncached.
 625  */
 626 static int
 627 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 628                          struct drm_i915_gem_pwrite *args,
 629                          struct drm_file *file_priv)
 630 {
 631         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 632         drm_i915_private_t *dev_priv = dev->dev_private;
 633         ssize_t remain;
 634         loff_t offset, page_base;
 635         char __user *user_data;
 636         int page_offset, page_length;
 637         int ret;
 638
 639         user_data = (char __user *) (uintptr_t) args->data_ptr;
 640         remain = args->size;
 641         if (!access_ok(VERIFY_READ, user_data, remain))
 642                 return -EFAULT;
 643
 644         ret = i915_mutex_lock_interruptible(dev);
 645         if (ret)
 646                 return ret;
 647
 648         ret = i915_gem_object_pin(obj, 0);
 649         if (ret) {
 650                 mutex_unlock(&dev->struct_mutex);
 651                 return ret;
 652         }
 653         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 654         if (ret)
 655                 goto fail;
 656
 657         obj_priv = to_intel_bo(obj);
 658         offset = obj_priv->gtt_offset + args->offset;
 659
 660         while (remain > 0) {
 661                 /* Operation in this page
 662                  *
 663                  * page_base = page offset within aperture
 664                  * page_offset = offset within page
 665                  * page_length = bytes to copy for this page
 666                  */
 667                 page_base = (offset & ~(PAGE_SIZE-1));
 668                 page_offset = offset & (PAGE_SIZE-1);
 669                 page_length = remain;
 670                 if ((page_offset + remain) > PAGE_SIZE)
 671                         page_length = PAGE_SIZE - page_offset;
 672
 673                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 674                                        page_offset, user_data, page_length);
 675
 676                 /* If we get a fault while copying data, then (presumably) our
 677                  * source page isn't available.  Return the error and we'll
 678                  * retry in the slow path.
 679                  */
 680                 if (ret)
 681                         goto fail;
 682
 683                 remain -= page_length;
 684                 user_data += page_length;
 685                 offset += page_length;
 686         }
 687
 688 fail:
 689         i915_gem_object_unpin(obj);
 690         mutex_unlock(&dev->struct_mutex);
 691
 692         return ret;
 693 }
 694
 695 /**
 696  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 697  * the memory and maps it using kmap_atomic for copying.
 698  *
 699  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 700  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 701  */
 702 static int
 703 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 704                          struct drm_i915_gem_pwrite *args,
 705                          struct drm_file *file_priv)
 706 {
 707         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 708         drm_i915_private_t *dev_priv = dev->dev_private;
 709         ssize_t remain;
 710         loff_t gtt_page_base, offset;
 711         loff_t first_data_page, last_data_page, num_pages;
 712         loff_t pinned_pages, i;
 713         struct page **user_pages;
 714         struct mm_struct *mm = current->mm;
 715         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 716         int ret;
 717         uint64_t data_ptr = args->data_ptr;
 718
 719         remain = args->size;
 720
 721         /* Pin the user pages containing the data.  We can't fault while
 722          * holding the struct mutex, and all of the pwrite implementations
 723          * want to hold it while dereferencing the user data.
 724          */
 725         first_data_page = data_ptr / PAGE_SIZE;
 726         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 727         num_pages = last_data_page - first_data_page + 1;
 728
 729         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 730         if (user_pages == NULL)
 731                 return -ENOMEM;
 732
 733         down_read(&mm->mmap_sem);
 734         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 735                                       num_pages, 0, 0, user_pages, NULL);
 736         up_read(&mm->mmap_sem);
 737         if (pinned_pages < num_pages) {
 738                 ret = -EFAULT;
 739                 goto out_unpin_pages;
 740         }
 741
 742         ret = i915_mutex_lock_interruptible(dev);
 743         if (ret)
 744                 goto out_unpin_pages;
 745
 746         ret = i915_gem_object_pin(obj, 0);
 747         if (ret)
 748                 goto out_unlock;
 749
 750         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 751         if (ret)
 752                 goto out_unpin_object;
 753
 754         obj_priv = to_intel_bo(obj);
 755         offset = obj_priv->gtt_offset + args->offset;
 756
 757         while (remain > 0) {
 758                 /* Operation in this page
 759                  *
 760                  * gtt_page_base = page offset within aperture
 761                  * gtt_page_offset = offset within page in aperture
 762                  * data_page_index = page number in get_user_pages return
 763                  * data_page_offset = offset with data_page_index page.
 764                  * page_length = bytes to copy for this page
 765                  */
 766                 gtt_page_base = offset & PAGE_MASK;
 767                 gtt_page_offset = offset & ~PAGE_MASK;
 768                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 769                 data_page_offset = data_ptr & ~PAGE_MASK;
 770
 771                 page_length = remain;
 772                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 773                         page_length = PAGE_SIZE - gtt_page_offset;
 774                 if ((data_page_offset + page_length) > PAGE_SIZE)
 775                         page_length = PAGE_SIZE - data_page_offset;
 776
 777                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 778                                   gtt_page_base, gtt_page_offset,
 779                                   user_pages[data_page_index],
 780                                   data_page_offset,
 781                                   page_length);
 782
 783                 remain -= page_length;
 784                 offset += page_length;
 785                 data_ptr += page_length;
 786         }
 787
 788 out_unpin_object:
 789         i915_gem_object_unpin(obj);
 790 out_unlock:
 791         mutex_unlock(&dev->struct_mutex);
 792 out_unpin_pages:
 793         for (i = 0; i < pinned_pages; i++)
 794                 page_cache_release(user_pages[i]);
 795         drm_free_large(user_pages);
 796
 797         return ret;
 798 }
 799
 800 /**
 801  * This is the fast shmem pwrite path, which attempts to directly
 802  * copy_from_user into the kmapped pages backing the object.
 803  */
 804 static int
 805 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 806                            struct drm_i915_gem_pwrite *args,
 807                            struct drm_file *file_priv)
 808 {
 809         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 810         ssize_t remain;
 811         loff_t offset, page_base;
 812         char __user *user_data;
 813         int page_offset, page_length;
 814         int ret;
 815
 816         user_data = (char __user *) (uintptr_t) args->data_ptr;
 817         remain = args->size;
 818
 819         ret = i915_mutex_lock_interruptible(dev);
 820         if (ret)
 821                 return ret;
 822
 823         ret = i915_gem_object_get_pages(obj, 0);
 824         if (ret != 0)
 825                 goto fail_unlock;
 826
 827         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 828         if (ret != 0)
 829                 goto fail_put_pages;
 830
 831         obj_priv = to_intel_bo(obj);
 832         offset = args->offset;
 833         obj_priv->dirty = 1;
 834
 835         while (remain > 0) {
 836                 /* Operation in this page
 837                  *
 838                  * page_base = page offset within aperture
 839                  * page_offset = offset within page
 840                  * page_length = bytes to copy for this page
 841                  */
 842                 page_base = (offset & ~(PAGE_SIZE-1));
 843                 page_offset = offset & (PAGE_SIZE-1);
 844                 page_length = remain;
 845                 if ((page_offset + remain) > PAGE_SIZE)
 846                         page_length = PAGE_SIZE - page_offset;
 847
 848                 ret = fast_shmem_write(obj_priv->pages,
 849                                        page_base, page_offset,
 850                                        user_data, page_length);
 851                 if (ret)
 852                         goto fail_put_pages;
 853
 854                 remain -= page_length;
 855                 user_data += page_length;
 856                 offset += page_length;
 857         }
 858
 859 fail_put_pages:
 860         i915_gem_object_put_pages(obj);
 861 fail_unlock:
 862         mutex_unlock(&dev->struct_mutex);
 863
 864         return ret;
 865 }
 866
 867 /**
 868  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 869  * the memory and maps it using kmap_atomic for copying.
 870  *
 871  * This avoids taking mmap_sem for faulting on the user's address while the
 872  * struct_mutex is held.
 873  */
 874 static int
 875 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 876                            struct drm_i915_gem_pwrite *args,
 877                            struct drm_file *file_priv)
 878 {
 879         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 880         struct mm_struct *mm = current->mm;
 881         struct page **user_pages;
 882         ssize_t remain;
 883         loff_t offset, pinned_pages, i;
 884         loff_t first_data_page, last_data_page, num_pages;
 885         int shmem_page_index, shmem_page_offset;
 886         int data_page_index,  data_page_offset;
 887         int page_length;
 888         int ret;
 889         uint64_t data_ptr = args->data_ptr;
 890         int do_bit17_swizzling;
 891
 892         remain = args->size;
 893
 894         /* Pin the user pages containing the data.  We can't fault while
 895          * holding the struct mutex, and all of the pwrite implementations
 896          * want to hold it while dereferencing the user data.
 897          */
 898         first_data_page = data_ptr / PAGE_SIZE;
 899         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 900         num_pages = last_data_page - first_data_page + 1;
 901
 902         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 903         if (user_pages == NULL)
 904                 return -ENOMEM;
 905
 906         down_read(&mm->mmap_sem);
 907         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 908                                       num_pages, 0, 0, user_pages, NULL);
 909         up_read(&mm->mmap_sem);
 910         if (pinned_pages < num_pages) {
 911                 ret = -EFAULT;
 912                 goto fail_put_user_pages;
 913         }
 914
 915         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 916
 917         ret = i915_mutex_lock_interruptible(dev);
 918         if (ret)
 919                 goto fail_put_user_pages;
 920
 921         ret = i915_gem_object_get_pages_or_evict(obj);
 922         if (ret)
 923                 goto fail_unlock;
 924
 925         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 926         if (ret != 0)
 927                 goto fail_put_pages;
 928
 929         obj_priv = to_intel_bo(obj);
 930         offset = args->offset;
 931         obj_priv->dirty = 1;
 932
 933         while (remain > 0) {
 934                 /* Operation in this page
 935                  *
 936                  * shmem_page_index = page number within shmem file
 937                  * shmem_page_offset = offset within page in shmem file
 938                  * data_page_index = page number in get_user_pages return
 939                  * data_page_offset = offset with data_page_index page.
 940                  * page_length = bytes to copy for this page
 941                  */
 942                 shmem_page_index = offset / PAGE_SIZE;
 943                 shmem_page_offset = offset & ~PAGE_MASK;
 944                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 945                 data_page_offset = data_ptr & ~PAGE_MASK;
 946
 947                 page_length = remain;
 948                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 949                         page_length = PAGE_SIZE - shmem_page_offset;
 950                 if ((data_page_offset + page_length) > PAGE_SIZE)
 951                         page_length = PAGE_SIZE - data_page_offset;
 952
 953                 if (do_bit17_swizzling) {
 954                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 955                                               shmem_page_offset,
 956                                               user_pages[data_page_index],
 957                                               data_page_offset,
 958                                               page_length,
 959                                               0);
 960                 } else {
 961                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 962                                         shmem_page_offset,
 963                                         user_pages[data_page_index],
 964                                         data_page_offset,
 965                                         page_length);
 966                 }
 967
 968                 remain -= page_length;
 969                 data_ptr += page_length;
 970                 offset += page_length;
 971         }
 972
 973 fail_put_pages:
 974         i915_gem_object_put_pages(obj);
 975 fail_unlock:
 976         mutex_unlock(&dev->struct_mutex);
 977 fail_put_user_pages:
 978         for (i = 0; i < pinned_pages; i++)
 979                 page_cache_release(user_pages[i]);
 980         drm_free_large(user_pages);
 981
 982         return ret;
 983 }
 984
 985 /**
 986  * Writes data to the object referenced by handle.
 987  *
 988  * On error, the contents of the buffer that were to be modified are undefined.
 989  */
 990 int
 991 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 992                       struct drm_file *file_priv)
 993 {
 994         struct drm_i915_gem_pwrite *args = data;
 995         struct drm_gem_object *obj;
 996         struct drm_i915_gem_object *obj_priv;
 997         int ret = 0;
 998
 999         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1000         if (obj == NULL)
1001                 return -ENOENT;
1002         obj_priv = to_intel_bo(obj);
1003
1004         /* Bounds check destination.
1005          *
1006          * XXX: This could use review for overflow issues...
1007          */
1008         if (args->offset > obj->size || args->size > obj->size ||
1009             args->offset + args->size > obj->size) {
1010                 drm_gem_object_unreference_unlocked(obj);
1011                 return -EINVAL;
1012         }
1013
1014         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1015          * it would end up going through the fenced access, and we'll get
1016          * different detiling behavior between reading and writing.
1017          * pread/pwrite currently are reading and writing from the CPU
1018          * perspective, requiring manual detiling by the client.
1019          */
1020         if (obj_priv->phys_obj)
1021                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
1022         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1023                  dev->gtt_total != 0 &&
1024                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1025                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
1026                 if (ret == -EFAULT) {
1027                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
1028                                                        file_priv);
1029                 }
1030         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
1031                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
1032         } else {
1033                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
1034                 if (ret == -EFAULT) {
1035                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
1036                                                          file_priv);
1037                 }
1038         }
1039
1040 #if WATCH_PWRITE
1041         if (ret)
1042                 DRM_INFO("pwrite failed %d\n", ret);
1043 #endif
1044
1045         drm_gem_object_unreference_unlocked(obj);
1046
1047         return ret;
1048 }
1049
1050 /**
1051  * Called when user space prepares to use an object with the CPU, either
1052  * through the mmap ioctl's mapping or a GTT mapping.
1053  */
1054 int
1055 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1056                           struct drm_file *file_priv)
1057 {
1058         struct drm_i915_private *dev_priv = dev->dev_private;
1059         struct drm_i915_gem_set_domain *args = data;
1060         struct drm_gem_object *obj;
1061         struct drm_i915_gem_object *obj_priv;
1062         uint32_t read_domains = args->read_domains;
1063         uint32_t write_domain = args->write_domain;
1064         int ret;
1065
1066         if (!(dev->driver->driver_features & DRIVER_GEM))
1067                 return -ENODEV;
1068
1069         /* Only handle setting domains to types used by the CPU. */
1070         if (write_domain & I915_GEM_GPU_DOMAINS)
1071                 return -EINVAL;
1072
1073         if (read_domains & I915_GEM_GPU_DOMAINS)
1074                 return -EINVAL;
1075
1076         /* Having something in the write domain implies it's in the read
1077          * domain, and only that read domain.  Enforce that in the request.
1078          */
1079         if (write_domain != 0 && read_domains != write_domain)
1080                 return -EINVAL;
1081
1082         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1083         if (obj == NULL)
1084                 return -ENOENT;
1085         obj_priv = to_intel_bo(obj);
1086
1087         ret = i915_mutex_lock_interruptible(dev);
1088         if (ret) {
1089                 drm_gem_object_unreference_unlocked(obj);
1090                 return ret;
1091         }
1092
1093         intel_mark_busy(dev, obj);
1094
1095 #if WATCH_BUF
1096         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1097                  obj, obj->size, read_domains, write_domain);
1098 #endif
1099         if (read_domains & I915_GEM_DOMAIN_GTT) {
1100                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1101
1102                 /* Update the LRU on the fence for the CPU access that's
1103                  * about to occur.
1104                  */
1105                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1106                         struct drm_i915_fence_reg *reg =
1107                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1108                         list_move_tail(&reg->lru_list,
1109                                        &dev_priv->mm.fence_list);
1110                 }
1111
1112                 /* Silently promote "you're not bound, there was nothing to do"
1113                  * to success, since the client was just asking us to
1114                  * make sure everything was done.
1115                  */
1116                 if (ret == -EINVAL)
1117                         ret = 0;
1118         } else {
1119                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1120         }
1121
1122         /* Maintain LRU order of "inactive" objects */
1123         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1124                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1125
1126         drm_gem_object_unreference(obj);
1127         mutex_unlock(&dev->struct_mutex);
1128         return ret;
1129 }
1130
1131 /**
1132  * Called when user space has done writes to this buffer
1133  */
1134 int
1135 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1136                       struct drm_file *file_priv)
1137 {
1138         struct drm_i915_gem_sw_finish *args = data;
1139         struct drm_gem_object *obj;
1140         struct drm_i915_gem_object *obj_priv;
1141         int ret = 0;
1142
1143         if (!(dev->driver->driver_features & DRIVER_GEM))
1144                 return -ENODEV;
1145
1146         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1147         if (obj == NULL)
1148                 return -ENOENT;
1149
1150         ret = i915_mutex_lock_interruptible(dev);
1151         if (ret) {
1152                 drm_gem_object_unreference_unlocked(obj);
1153                 return ret;
1154         }
1155
1156 #if WATCH_BUF
1157         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1158                  __func__, args->handle, obj, obj->size);
1159 #endif
1160         obj_priv = to_intel_bo(obj);
1161
1162         /* Pinned buffers may be scanout, so flush the cache */
1163         if (obj_priv->pin_count)
1164                 i915_gem_object_flush_cpu_write_domain(obj);
1165
1166         drm_gem_object_unreference(obj);
1167         mutex_unlock(&dev->struct_mutex);
1168         return ret;
1169 }
1170
1171 /**
1172  * Maps the contents of an object, returning the address it is mapped
1173  * into.
1174  *
1175  * While the mapping holds a reference on the contents of the object, it doesn't
1176  * imply a ref on the object itself.
1177  */
1178 int
1179 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1180                    struct drm_file *file_priv)
1181 {
1182         struct drm_i915_gem_mmap *args = data;
1183         struct drm_gem_object *obj;
1184         loff_t offset;
1185         unsigned long addr;
1186
1187         if (!(dev->driver->driver_features & DRIVER_GEM))
1188                 return -ENODEV;
1189
1190         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1191         if (obj == NULL)
1192                 return -ENOENT;
1193
1194         offset = args->offset;
1195
1196         down_write(&current->mm->mmap_sem);
1197         addr = do_mmap(obj->filp, 0, args->size,
1198                        PROT_READ | PROT_WRITE, MAP_SHARED,
1199                        args->offset);
1200         up_write(&current->mm->mmap_sem);
1201         drm_gem_object_unreference_unlocked(obj);
1202         if (IS_ERR((void *)addr))
1203                 return addr;
1204
1205         args->addr_ptr = (uint64_t) addr;
1206
1207         return 0;
1208 }
1209
1210 /**
1211  * i915_gem_fault - fault a page into the GTT
1212  * vma: VMA in question
1213  * vmf: fault info
1214  *
1215  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1216  * from userspace.  The fault handler takes care of binding the object to
1217  * the GTT (if needed), allocating and programming a fence register (again,
1218  * only if needed based on whether the old reg is still valid or the object
1219  * is tiled) and inserting a new PTE into the faulting process.
1220  *
1221  * Note that the faulting process may involve evicting existing objects
1222  * from the GTT and/or fence registers to make room.  So performance may
1223  * suffer if the GTT working set is large or there are few fence registers
1224  * left.
1225  */
1226 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1227 {
1228         struct drm_gem_object *obj = vma->vm_private_data;
1229         struct drm_device *dev = obj->dev;
1230         drm_i915_private_t *dev_priv = dev->dev_private;
1231         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1232         pgoff_t page_offset;
1233         unsigned long pfn;
1234         int ret = 0;
1235         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1236
1237         /* We don't use vmf->pgoff since that has the fake offset */
1238         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1239                 PAGE_SHIFT;
1240
1241         /* Now bind it into the GTT if needed */
1242         mutex_lock(&dev->struct_mutex);
1243         if (!obj_priv->gtt_space) {
1244                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1245                 if (ret)
1246                         goto unlock;
1247
1248                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1249                 if (ret)
1250                         goto unlock;
1251         }
1252
1253         /* Need a new fence register? */
1254         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1255                 ret = i915_gem_object_get_fence_reg(obj, true);
1256                 if (ret)
1257                         goto unlock;
1258         }
1259
1260         if (i915_gem_object_is_inactive(obj_priv))
1261                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1262
1263         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1264                 page_offset;
1265
1266         /* Finally, remap it using the new GTT offset */
1267         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1268 unlock:
1269         mutex_unlock(&dev->struct_mutex);
1270
1271         switch (ret) {
1272         case 0:
1273         case -ERESTARTSYS:
1274                 return VM_FAULT_NOPAGE;
1275         case -ENOMEM:
1276         case -EAGAIN:
1277                 return VM_FAULT_OOM;
1278         default:
1279                 return VM_FAULT_SIGBUS;
1280         }
1281 }
1282
1283 /**
1284  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1285  * @obj: obj in question
1286  *
1287  * GEM memory mapping works by handing back to userspace a fake mmap offset
1288  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1289  * up the object based on the offset and sets up the various memory mapping
1290  * structures.
1291  *
1292  * This routine allocates and attaches a fake offset for @obj.
1293  */
1294 static int
1295 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1296 {
1297         struct drm_device *dev = obj->dev;
1298         struct drm_gem_mm *mm = dev->mm_private;
1299         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1300         struct drm_map_list *list;
1301         struct drm_local_map *map;
1302         int ret = 0;
1303
1304         /* Set the object up for mmap'ing */
1305         list = &obj->map_list;
1306         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1307         if (!list->map)
1308                 return -ENOMEM;
1309
1310         map = list->map;
1311         map->type = _DRM_GEM;
1312         map->size = obj->size;
1313         map->handle = obj;
1314
1315         /* Get a DRM GEM mmap offset allocated... */
1316         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1317                                                     obj->size / PAGE_SIZE, 0, 0);
1318         if (!list->file_offset_node) {
1319                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1320                 ret = -ENOSPC;
1321                 goto out_free_list;
1322         }
1323
1324         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1325                                                   obj->size / PAGE_SIZE, 0);
1326         if (!list->file_offset_node) {
1327                 ret = -ENOMEM;
1328                 goto out_free_list;
1329         }
1330
1331         list->hash.key = list->file_offset_node->start;
1332         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1333         if (ret) {
1334                 DRM_ERROR("failed to add to map hash\n");
1335                 goto out_free_mm;
1336         }
1337
1338         /* By now we should be all set, any drm_mmap request on the offset
1339          * below will get to our mmap & fault handler */
1340         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1341
1342         return 0;
1343
1344 out_free_mm:
1345         drm_mm_put_block(list->file_offset_node);
1346 out_free_list:
1347         kfree(list->map);
1348
1349         return ret;
1350 }
1351
1352 /**
1353  * i915_gem_release_mmap - remove physical page mappings
1354  * @obj: obj in question
1355  *
1356  * Preserve the reservation of the mmapping with the DRM core code, but
1357  * relinquish ownership of the pages back to the system.
1358  *
1359  * It is vital that we remove the page mapping if we have mapped a tiled
1360  * object through the GTT and then lose the fence register due to
1361  * resource pressure. Similarly if the object has been moved out of the
1362  * aperture, than pages mapped into userspace must be revoked. Removing the
1363  * mapping will then trigger a page fault on the next user access, allowing
1364  * fixup by i915_gem_fault().
1365  */
1366 void
1367 i915_gem_release_mmap(struct drm_gem_object *obj)
1368 {
1369         struct drm_device *dev = obj->dev;
1370         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1371
1372         if (dev->dev_mapping)
1373                 unmap_mapping_range(dev->dev_mapping,
1374                                     obj_priv->mmap_offset, obj->size, 1);
1375 }
1376
1377 static void
1378 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1379 {
1380         struct drm_device *dev = obj->dev;
1381         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1382         struct drm_gem_mm *mm = dev->mm_private;
1383         struct drm_map_list *list;
1384
1385         list = &obj->map_list;
1386         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1387
1388         if (list->file_offset_node) {
1389                 drm_mm_put_block(list->file_offset_node);
1390                 list->file_offset_node = NULL;
1391         }
1392
1393         if (list->map) {
1394                 kfree(list->map);
1395                 list->map = NULL;
1396         }
1397
1398         obj_priv->mmap_offset = 0;
1399 }
1400
1401 /**
1402  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1403  * @obj: object to check
1404  *
1405  * Return the required GTT alignment for an object, taking into account
1406  * potential fence register mapping if needed.
1407  */
1408 static uint32_t
1409 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1410 {
1411         struct drm_device *dev = obj->dev;
1412         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1413         int start, i;
1414
1415         /*
1416          * Minimum alignment is 4k (GTT page size), but might be greater
1417          * if a fence register is needed for the object.
1418          */
1419         if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
1420                 return 4096;
1421
1422         /*
1423          * Previous chips need to be aligned to the size of the smallest
1424          * fence register that can contain the object.
1425          */
1426         if (INTEL_INFO(dev)->gen == 3)
1427                 start = 1024*1024;
1428         else
1429                 start = 512*1024;
1430
1431         for (i = start; i < obj->size; i <<= 1)
1432                 ;
1433
1434         return i;
1435 }
1436
1437 /**
1438  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1439  * @dev: DRM device
1440  * @data: GTT mapping ioctl data
1441  * @file_priv: GEM object info
1442  *
1443  * Simply returns the fake offset to userspace so it can mmap it.
1444  * The mmap call will end up in drm_gem_mmap(), which will set things
1445  * up so we can get faults in the handler above.
1446  *
1447  * The fault handler will take care of binding the object into the GTT
1448  * (since it may have been evicted to make room for something), allocating
1449  * a fence register, and mapping the appropriate aperture address into
1450  * userspace.
1451  */
1452 int
1453 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1454                         struct drm_file *file_priv)
1455 {
1456         struct drm_i915_gem_mmap_gtt *args = data;
1457         struct drm_gem_object *obj;
1458         struct drm_i915_gem_object *obj_priv;
1459         int ret;
1460
1461         if (!(dev->driver->driver_features & DRIVER_GEM))
1462                 return -ENODEV;
1463
1464         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1465         if (obj == NULL)
1466                 return -ENOENT;
1467
1468         ret = i915_mutex_lock_interruptible(dev);
1469         if (ret) {
1470                 drm_gem_object_unreference_unlocked(obj);
1471                 return ret;
1472         }
1473
1474         obj_priv = to_intel_bo(obj);
1475
1476         if (obj_priv->madv != I915_MADV_WILLNEED) {
1477                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1478                 drm_gem_object_unreference(obj);
1479                 mutex_unlock(&dev->struct_mutex);
1480                 return -EINVAL;
1481         }
1482
1483
1484         if (!obj_priv->mmap_offset) {
1485                 ret = i915_gem_create_mmap_offset(obj);
1486                 if (ret) {
1487                         drm_gem_object_unreference(obj);
1488                         mutex_unlock(&dev->struct_mutex);
1489                         return ret;
1490                 }
1491         }
1492
1493         args->offset = obj_priv->mmap_offset;
1494
1495         /*
1496          * Pull it into the GTT so that we have a page list (makes the
1497          * initial fault faster and any subsequent flushing possible).
1498          */
1499         if (!obj_priv->agp_mem) {
1500                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1501                 if (ret) {
1502                         drm_gem_object_unreference(obj);
1503                         mutex_unlock(&dev->struct_mutex);
1504                         return ret;
1505                 }
1506         }
1507
1508         drm_gem_object_unreference(obj);
1509         mutex_unlock(&dev->struct_mutex);
1510
1511         return 0;
1512 }
1513
1514 void
1515 i915_gem_object_put_pages(struct drm_gem_object *obj)
1516 {
1517         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1518         int page_count = obj->size / PAGE_SIZE;
1519         int i;
1520
1521         BUG_ON(obj_priv->pages_refcount == 0);
1522         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1523
1524         if (--obj_priv->pages_refcount != 0)
1525                 return;
1526
1527         if (obj_priv->tiling_mode != I915_TILING_NONE)
1528                 i915_gem_object_save_bit_17_swizzle(obj);
1529
1530         if (obj_priv->madv == I915_MADV_DONTNEED)
1531                 obj_priv->dirty = 0;
1532
1533         for (i = 0; i < page_count; i++) {
1534                 if (obj_priv->dirty)
1535                         set_page_dirty(obj_priv->pages[i]);
1536
1537                 if (obj_priv->madv == I915_MADV_WILLNEED)
1538                         mark_page_accessed(obj_priv->pages[i]);
1539
1540                 page_cache_release(obj_priv->pages[i]);
1541         }
1542         obj_priv->dirty = 0;
1543
1544         drm_free_large(obj_priv->pages);
1545         obj_priv->pages = NULL;
1546 }
1547
1548 static void
1549 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1550                                struct intel_ring_buffer *ring)
1551 {
1552         struct drm_i915_private *dev_priv = obj->dev->dev_private;
1553         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1554
1555         BUG_ON(ring == NULL);
1556         obj_priv->ring = ring;
1557
1558         /* Add a reference if we're newly entering the active list. */
1559         if (!obj_priv->active) {
1560                 drm_gem_object_reference(obj);
1561                 obj_priv->active = 1;
1562         }
1563
1564         /* Move from whatever list we were on to the tail of execution. */
1565         list_move_tail(&obj_priv->list, &ring->active_list);
1566         obj_priv->last_rendering_seqno = dev_priv->next_seqno;
1567 }
1568
1569 static void
1570 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1571 {
1572         struct drm_device *dev = obj->dev;
1573         drm_i915_private_t *dev_priv = dev->dev_private;
1574         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1575
1576         BUG_ON(!obj_priv->active);
1577         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1578         obj_priv->last_rendering_seqno = 0;
1579 }
1580
1581 /* Immediately discard the backing storage */
1582 static void
1583 i915_gem_object_truncate(struct drm_gem_object *obj)
1584 {
1585         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1586         struct inode *inode;
1587
1588         /* Our goal here is to return as much of the memory as
1589          * is possible back to the system as we are called from OOM.
1590          * To do this we must instruct the shmfs to drop all of its
1591          * backing pages, *now*. Here we mirror the actions taken
1592          * when by shmem_delete_inode() to release the backing store.
1593          */
1594         inode = obj->filp->f_path.dentry->d_inode;
1595         truncate_inode_pages(inode->i_mapping, 0);
1596         if (inode->i_op->truncate_range)
1597                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1598
1599         obj_priv->madv = __I915_MADV_PURGED;
1600 }
1601
1602 static inline int
1603 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1604 {
1605         return obj_priv->madv == I915_MADV_DONTNEED;
1606 }
1607
1608 static void
1609 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1610 {
1611         struct drm_device *dev = obj->dev;
1612         drm_i915_private_t *dev_priv = dev->dev_private;
1613         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1614
1615         i915_verify_inactive(dev, __FILE__, __LINE__);
1616         if (obj_priv->pin_count != 0)
1617                 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list);
1618         else
1619                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1620
1621         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1622
1623         obj_priv->last_rendering_seqno = 0;
1624         obj_priv->ring = NULL;
1625         if (obj_priv->active) {
1626                 obj_priv->active = 0;
1627                 drm_gem_object_unreference(obj);
1628         }
1629         i915_verify_inactive(dev, __FILE__, __LINE__);
1630 }
1631
1632 static void
1633 i915_gem_process_flushing_list(struct drm_device *dev,
1634                                uint32_t flush_domains,
1635                                struct intel_ring_buffer *ring)
1636 {
1637         drm_i915_private_t *dev_priv = dev->dev_private;
1638         struct drm_i915_gem_object *obj_priv, *next;
1639
1640         list_for_each_entry_safe(obj_priv, next,
1641                                  &dev_priv->mm.gpu_write_list,
1642                                  gpu_write_list) {
1643                 struct drm_gem_object *obj = &obj_priv->base;
1644
1645                 if (obj->write_domain & flush_domains &&
1646                     obj_priv->ring == ring) {
1647                         uint32_t old_write_domain = obj->write_domain;
1648
1649                         obj->write_domain = 0;
1650                         list_del_init(&obj_priv->gpu_write_list);
1651                         i915_gem_object_move_to_active(obj, ring);
1652
1653                         /* update the fence lru list */
1654                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1655                                 struct drm_i915_fence_reg *reg =
1656                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1657                                 list_move_tail(&reg->lru_list,
1658                                                 &dev_priv->mm.fence_list);
1659                         }
1660
1661                         trace_i915_gem_object_change_domain(obj,
1662                                                             obj->read_domains,
1663                                                             old_write_domain);
1664                 }
1665         }
1666 }
1667
1668 uint32_t
1669 i915_add_request(struct drm_device *dev,
1670                  struct drm_file *file,
1671                  struct drm_i915_gem_request *request,
1672                  struct intel_ring_buffer *ring)
1673 {
1674         drm_i915_private_t *dev_priv = dev->dev_private;
1675         struct drm_i915_file_private *file_priv = NULL;
1676         uint32_t seqno;
1677         int was_empty;
1678
1679         if (file != NULL)
1680                 file_priv = file->driver_priv;
1681
1682         if (request == NULL) {
1683                 request = kzalloc(sizeof(*request), GFP_KERNEL);
1684                 if (request == NULL)
1685                         return 0;
1686         }
1687
1688         seqno = ring->add_request(dev, ring, 0);
1689
1690         request->seqno = seqno;
1691         request->ring = ring;
1692         request->emitted_jiffies = jiffies;
1693         was_empty = list_empty(&ring->request_list);
1694         list_add_tail(&request->list, &ring->request_list);
1695
1696         if (file_priv) {
1697                 mutex_lock(&file_priv->mutex);
1698                 request->file_priv = file_priv;
1699                 list_add_tail(&request->client_list,
1700                               &file_priv->mm.request_list);
1701                 mutex_unlock(&file_priv->mutex);
1702         }
1703
1704         if (!dev_priv->mm.suspended) {
1705                 mod_timer(&dev_priv->hangcheck_timer,
1706                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1707                 if (was_empty)
1708                         queue_delayed_work(dev_priv->wq,
1709                                            &dev_priv->mm.retire_work, HZ);
1710         }
1711         return seqno;
1712 }
1713
1714 /**
1715  * Command execution barrier
1716  *
1717  * Ensures that all commands in the ring are finished
1718  * before signalling the CPU
1719  */
1720 static void
1721 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1722 {
1723         uint32_t flush_domains = 0;
1724
1725         /* The sampler always gets flushed on i965 (sigh) */
1726         if (INTEL_INFO(dev)->gen >= 4)
1727                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1728
1729         ring->flush(dev, ring,
1730                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1731 }
1732
1733 static inline void
1734 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1735 {
1736         if (request->file_priv) {
1737                 mutex_lock(&request->file_priv->mutex);
1738                 list_del(&request->client_list);
1739                 mutex_unlock(&request->file_priv->mutex);
1740         }
1741 }
1742
1743 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1744                                       struct intel_ring_buffer *ring)
1745 {
1746         while (!list_empty(&ring->request_list)) {
1747                 struct drm_i915_gem_request *request;
1748
1749                 request = list_first_entry(&ring->request_list,
1750                                            struct drm_i915_gem_request,
1751                                            list);
1752
1753                 list_del(&request->list);
1754                 i915_gem_request_remove_from_client(request);
1755                 kfree(request);
1756         }
1757
1758         while (!list_empty(&ring->active_list)) {
1759                 struct drm_i915_gem_object *obj_priv;
1760
1761                 obj_priv = list_first_entry(&ring->active_list,
1762                                             struct drm_i915_gem_object,
1763                                             list);
1764
1765                 obj_priv->base.write_domain = 0;
1766                 list_del_init(&obj_priv->gpu_write_list);
1767                 i915_gem_object_move_to_inactive(&obj_priv->base);
1768         }
1769 }
1770
1771 void i915_gem_reset_lists(struct drm_device *dev)
1772 {
1773         struct drm_i915_private *dev_priv = dev->dev_private;
1774         struct drm_i915_gem_object *obj_priv;
1775
1776         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1777         if (HAS_BSD(dev))
1778                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1779
1780         /* Remove anything from the flushing lists. The GPU cache is likely
1781          * to be lost on reset along with the data, so simply move the
1782          * lost bo to the inactive list.
1783          */
1784         while (!list_empty(&dev_priv->mm.flushing_list)) {
1785                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1786                                             struct drm_i915_gem_object,
1787                                             list);
1788
1789                 obj_priv->base.write_domain = 0;
1790                 list_del_init(&obj_priv->gpu_write_list);
1791                 i915_gem_object_move_to_inactive(&obj_priv->base);
1792         }
1793
1794         /* Move everything out of the GPU domains to ensure we do any
1795          * necessary invalidation upon reuse.
1796          */
1797         list_for_each_entry(obj_priv,
1798                             &dev_priv->mm.inactive_list,
1799                             list)
1800         {
1801                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1802         }
1803 }
1804
1805 /**
1806  * This function clears the request list as sequence numbers are passed.
1807  */
1808 static void
1809 i915_gem_retire_requests_ring(struct drm_device *dev,
1810                               struct intel_ring_buffer *ring)
1811 {
1812         drm_i915_private_t *dev_priv = dev->dev_private;
1813         uint32_t seqno;
1814
1815         if (!ring->status_page.page_addr ||
1816             list_empty(&ring->request_list))
1817                 return;
1818
1819         seqno = ring->get_seqno(dev, ring);
1820         while (!list_empty(&ring->request_list)) {
1821                 struct drm_i915_gem_request *request;
1822
1823                 request = list_first_entry(&ring->request_list,
1824                                            struct drm_i915_gem_request,
1825                                            list);
1826
1827                 if (!i915_seqno_passed(seqno, request->seqno))
1828                         break;
1829
1830                 trace_i915_gem_request_retire(dev, request->seqno);
1831
1832                 list_del(&request->list);
1833                 i915_gem_request_remove_from_client(request);
1834                 kfree(request);
1835         }
1836
1837         /* Move any buffers on the active list that are no longer referenced
1838          * by the ringbuffer to the flushing/inactive lists as appropriate.
1839          */
1840         while (!list_empty(&ring->active_list)) {
1841                 struct drm_gem_object *obj;
1842                 struct drm_i915_gem_object *obj_priv;
1843
1844                 obj_priv = list_first_entry(&ring->active_list,
1845                                             struct drm_i915_gem_object,
1846                                             list);
1847
1848                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1849                         break;
1850
1851                 obj = &obj_priv->base;
1852
1853 #if WATCH_LRU
1854                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1855                          __func__, request->seqno, obj);
1856 #endif
1857
1858                 if (obj->write_domain != 0)
1859                         i915_gem_object_move_to_flushing(obj);
1860                 else
1861                         i915_gem_object_move_to_inactive(obj);
1862         }
1863
1864         if (unlikely (dev_priv->trace_irq_seqno &&
1865                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1866                 ring->user_irq_put(dev, ring);
1867                 dev_priv->trace_irq_seqno = 0;
1868         }
1869 }
1870
1871 void
1872 i915_gem_retire_requests(struct drm_device *dev)
1873 {
1874         drm_i915_private_t *dev_priv = dev->dev_private;
1875
1876         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1877             struct drm_i915_gem_object *obj_priv, *tmp;
1878
1879             /* We must be careful that during unbind() we do not
1880              * accidentally infinitely recurse into retire requests.
1881              * Currently:
1882              *   retire -> free -> unbind -> wait -> retire_ring
1883              */
1884             list_for_each_entry_safe(obj_priv, tmp,
1885                                      &dev_priv->mm.deferred_free_list,
1886                                      list)
1887                     i915_gem_free_object_tail(&obj_priv->base);
1888         }
1889
1890         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1891         if (HAS_BSD(dev))
1892                 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1893 }
1894
1895 static void
1896 i915_gem_retire_work_handler(struct work_struct *work)
1897 {
1898         drm_i915_private_t *dev_priv;
1899         struct drm_device *dev;
1900
1901         dev_priv = container_of(work, drm_i915_private_t,
1902                                 mm.retire_work.work);
1903         dev = dev_priv->dev;
1904
1905         mutex_lock(&dev->struct_mutex);
1906         i915_gem_retire_requests(dev);
1907
1908         if (!dev_priv->mm.suspended &&
1909                 (!list_empty(&dev_priv->render_ring.request_list) ||
1910                         (HAS_BSD(dev) &&
1911                          !list_empty(&dev_priv->bsd_ring.request_list))))
1912                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1913         mutex_unlock(&dev->struct_mutex);
1914 }
1915
1916 int
1917 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1918                      bool interruptible, struct intel_ring_buffer *ring)
1919 {
1920         drm_i915_private_t *dev_priv = dev->dev_private;
1921         u32 ier;
1922         int ret = 0;
1923
1924         BUG_ON(seqno == 0);
1925
1926         if (atomic_read(&dev_priv->mm.wedged))
1927                 return -EAGAIN;
1928
1929         if (seqno == dev_priv->next_seqno) {
1930                 seqno = i915_add_request(dev, NULL, NULL, ring);
1931                 if (seqno == 0)
1932                         return -ENOMEM;
1933         }
1934
1935         if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
1936                 if (HAS_PCH_SPLIT(dev))
1937                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1938                 else
1939                         ier = I915_READ(IER);
1940                 if (!ier) {
1941                         DRM_ERROR("something (likely vbetool) disabled "
1942                                   "interrupts, re-enabling\n");
1943                         i915_driver_irq_preinstall(dev);
1944                         i915_driver_irq_postinstall(dev);
1945                 }
1946
1947                 trace_i915_gem_request_wait_begin(dev, seqno);
1948
1949                 ring->waiting_gem_seqno = seqno;
1950                 ring->user_irq_get(dev, ring);
1951                 if (interruptible)
1952                         ret = wait_event_interruptible(ring->irq_queue,
1953                                 i915_seqno_passed(
1954                                         ring->get_seqno(dev, ring), seqno)
1955                                 || atomic_read(&dev_priv->mm.wedged));
1956                 else
1957                         wait_event(ring->irq_queue,
1958                                 i915_seqno_passed(
1959                                         ring->get_seqno(dev, ring), seqno)
1960                                 || atomic_read(&dev_priv->mm.wedged));
1961
1962                 ring->user_irq_put(dev, ring);
1963                 ring->waiting_gem_seqno = 0;
1964
1965                 trace_i915_gem_request_wait_end(dev, seqno);
1966         }
1967         if (atomic_read(&dev_priv->mm.wedged))
1968                 ret = -EAGAIN;
1969
1970         if (ret && ret != -ERESTARTSYS)
1971                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
1972                           __func__, ret, seqno, ring->get_seqno(dev, ring),
1973                           dev_priv->next_seqno);
1974
1975         /* Directly dispatch request retiring.  While we have the work queue
1976          * to handle this, the waiter on a request often wants an associated
1977          * buffer to have made it to the inactive list, and we would need
1978          * a separate wait queue to handle that.
1979          */
1980         if (ret == 0)
1981                 i915_gem_retire_requests_ring(dev, ring);
1982
1983         return ret;
1984 }
1985
1986 /**
1987  * Waits for a sequence number to be signaled, and cleans up the
1988  * request and object lists appropriately for that event.
1989  */
1990 static int
1991 i915_wait_request(struct drm_device *dev, uint32_t seqno,
1992                 struct intel_ring_buffer *ring)
1993 {
1994         return i915_do_wait_request(dev, seqno, 1, ring);
1995 }
1996
1997 static void
1998 i915_gem_flush_ring(struct drm_device *dev,
1999                     struct drm_file *file_priv,
2000                     struct intel_ring_buffer *ring,
2001                     uint32_t invalidate_domains,
2002                     uint32_t flush_domains)
2003 {
2004         ring->flush(dev, ring, invalidate_domains, flush_domains);
2005         i915_gem_process_flushing_list(dev, flush_domains, ring);
2006 }
2007
2008 static void
2009 i915_gem_flush(struct drm_device *dev,
2010                struct drm_file *file_priv,
2011                uint32_t invalidate_domains,
2012                uint32_t flush_domains,
2013                uint32_t flush_rings)
2014 {
2015         drm_i915_private_t *dev_priv = dev->dev_private;
2016
2017         if (flush_domains & I915_GEM_DOMAIN_CPU)
2018                 drm_agp_chipset_flush(dev);
2019
2020         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2021                 if (flush_rings & RING_RENDER)
2022                         i915_gem_flush_ring(dev, file_priv,
2023                                             &dev_priv->render_ring,
2024                                             invalidate_domains, flush_domains);
2025                 if (flush_rings & RING_BSD)
2026                         i915_gem_flush_ring(dev, file_priv,
2027                                             &dev_priv->bsd_ring,
2028                                             invalidate_domains, flush_domains);
2029         }
2030 }
2031
2032 /**
2033  * Ensures that all rendering to the object has completed and the object is
2034  * safe to unbind from the GTT or access from the CPU.
2035  */
2036 static int
2037 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2038                                bool interruptible)
2039 {
2040         struct drm_device *dev = obj->dev;
2041         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2042         int ret;
2043
2044         /* This function only exists to support waiting for existing rendering,
2045          * not for emitting required flushes.
2046          */
2047         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2048
2049         /* If there is rendering queued on the buffer being evicted, wait for
2050          * it.
2051          */
2052         if (obj_priv->active) {
2053 #if WATCH_BUF
2054                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2055                           __func__, obj, obj_priv->last_rendering_seqno);
2056 #endif
2057                 ret = i915_do_wait_request(dev,
2058                                            obj_priv->last_rendering_seqno,
2059                                            interruptible,
2060                                            obj_priv->ring);
2061                 if (ret)
2062                         return ret;
2063         }
2064
2065         return 0;
2066 }
2067
2068 /**
2069  * Unbinds an object from the GTT aperture.
2070  */
2071 int
2072 i915_gem_object_unbind(struct drm_gem_object *obj)
2073 {
2074         struct drm_device *dev = obj->dev;
2075         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2076         int ret = 0;
2077
2078 #if WATCH_BUF
2079         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2080         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2081 #endif
2082         if (obj_priv->gtt_space == NULL)
2083                 return 0;
2084
2085         if (obj_priv->pin_count != 0) {
2086                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2087                 return -EINVAL;
2088         }
2089
2090         /* blow away mappings if mapped through GTT */
2091         i915_gem_release_mmap(obj);
2092
2093         /* Move the object to the CPU domain to ensure that
2094          * any possible CPU writes while it's not in the GTT
2095          * are flushed when we go to remap it. This will
2096          * also ensure that all pending GPU writes are finished
2097          * before we unbind.
2098          */
2099         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2100         if (ret == -ERESTARTSYS)
2101                 return ret;
2102         /* Continue on if we fail due to EIO, the GPU is hung so we
2103          * should be safe and we need to cleanup or else we might
2104          * cause memory corruption through use-after-free.
2105          */
2106
2107         /* release the fence reg _after_ flushing */
2108         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2109                 i915_gem_clear_fence_reg(obj);
2110
2111         if (obj_priv->agp_mem != NULL) {
2112                 drm_unbind_agp(obj_priv->agp_mem);
2113                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2114                 obj_priv->agp_mem = NULL;
2115         }
2116
2117         i915_gem_object_put_pages(obj);
2118         BUG_ON(obj_priv->pages_refcount);
2119
2120         if (obj_priv->gtt_space) {
2121                 atomic_dec(&dev->gtt_count);
2122                 atomic_sub(obj->size, &dev->gtt_memory);
2123
2124                 drm_mm_put_block(obj_priv->gtt_space);
2125                 obj_priv->gtt_space = NULL;
2126         }
2127
2128         list_del_init(&obj_priv->list);
2129
2130         if (i915_gem_object_is_purgeable(obj_priv))
2131                 i915_gem_object_truncate(obj);
2132
2133         trace_i915_gem_object_unbind(obj);
2134
2135         return ret;
2136 }
2137
2138 int
2139 i915_gpu_idle(struct drm_device *dev)
2140 {
2141         drm_i915_private_t *dev_priv = dev->dev_private;
2142         bool lists_empty;
2143         u32 seqno;
2144         int ret;
2145
2146         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2147                        list_empty(&dev_priv->render_ring.active_list) &&
2148                        (!HAS_BSD(dev) ||
2149                         list_empty(&dev_priv->bsd_ring.active_list)));
2150         if (lists_empty)
2151                 return 0;
2152
2153         /* Flush everything onto the inactive list. */
2154         seqno = dev_priv->next_seqno;
2155         i915_gem_flush_ring(dev, NULL, &dev_priv->render_ring,
2156                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2157         ret = i915_wait_request(dev, seqno, &dev_priv->render_ring);
2158         if (ret)
2159                 return ret;
2160
2161         if (HAS_BSD(dev)) {
2162                 seqno = dev_priv->next_seqno;
2163                 i915_gem_flush_ring(dev, NULL, &dev_priv->bsd_ring,
2164                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2165                 ret = i915_wait_request(dev, seqno, &dev_priv->bsd_ring);
2166                 if (ret)
2167                         return ret;
2168         }
2169
2170         return 0;
2171 }
2172
2173 int
2174 i915_gem_object_get_pages(struct drm_gem_object *obj,
2175                           gfp_t gfpmask)
2176 {
2177         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2178         int page_count, i;
2179         struct address_space *mapping;
2180         struct inode *inode;
2181         struct page *page;
2182
2183         BUG_ON(obj_priv->pages_refcount
2184                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2185
2186         if (obj_priv->pages_refcount++ != 0)
2187                 return 0;
2188
2189         /* Get the list of pages out of our struct file.  They'll be pinned
2190          * at this point until we release them.
2191          */
2192         page_count = obj->size / PAGE_SIZE;
2193         BUG_ON(obj_priv->pages != NULL);
2194         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2195         if (obj_priv->pages == NULL) {
2196                 obj_priv->pages_refcount--;
2197                 return -ENOMEM;
2198         }
2199
2200         inode = obj->filp->f_path.dentry->d_inode;
2201         mapping = inode->i_mapping;
2202         for (i = 0; i < page_count; i++) {
2203                 page = read_cache_page_gfp(mapping, i,
2204                                            GFP_HIGHUSER |
2205                                            __GFP_COLD |
2206                                            __GFP_RECLAIMABLE |
2207                                            gfpmask);
2208                 if (IS_ERR(page))
2209                         goto err_pages;
2210
2211                 obj_priv->pages[i] = page;
2212         }
2213
2214         if (obj_priv->tiling_mode != I915_TILING_NONE)
2215                 i915_gem_object_do_bit_17_swizzle(obj);
2216
2217         return 0;
2218
2219 err_pages:
2220         while (i--)
2221                 page_cache_release(obj_priv->pages[i]);
2222
2223         drm_free_large(obj_priv->pages);
2224         obj_priv->pages = NULL;
2225         obj_priv->pages_refcount--;
2226         return PTR_ERR(page);
2227 }
2228
2229 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2230 {
2231         struct drm_gem_object *obj = reg->obj;
2232         struct drm_device *dev = obj->dev;
2233         drm_i915_private_t *dev_priv = dev->dev_private;
2234         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2235         int regnum = obj_priv->fence_reg;
2236         uint64_t val;
2237
2238         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2239                     0xfffff000) << 32;
2240         val |= obj_priv->gtt_offset & 0xfffff000;
2241         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2242                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2243
2244         if (obj_priv->tiling_mode == I915_TILING_Y)
2245                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2246         val |= I965_FENCE_REG_VALID;
2247
2248         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2249 }
2250
2251 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2252 {
2253         struct drm_gem_object *obj = reg->obj;
2254         struct drm_device *dev = obj->dev;
2255         drm_i915_private_t *dev_priv = dev->dev_private;
2256         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2257         int regnum = obj_priv->fence_reg;
2258         uint64_t val;
2259
2260         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2261                     0xfffff000) << 32;
2262         val |= obj_priv->gtt_offset & 0xfffff000;
2263         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2264         if (obj_priv->tiling_mode == I915_TILING_Y)
2265                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2266         val |= I965_FENCE_REG_VALID;
2267
2268         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2269 }
2270
2271 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2272 {
2273         struct drm_gem_object *obj = reg->obj;
2274         struct drm_device *dev = obj->dev;
2275         drm_i915_private_t *dev_priv = dev->dev_private;
2276         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2277         int regnum = obj_priv->fence_reg;
2278         int tile_width;
2279         uint32_t fence_reg, val;
2280         uint32_t pitch_val;
2281
2282         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2283             (obj_priv->gtt_offset & (obj->size - 1))) {
2284                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2285                      __func__, obj_priv->gtt_offset, obj->size);
2286                 return;
2287         }
2288
2289         if (obj_priv->tiling_mode == I915_TILING_Y &&
2290             HAS_128_BYTE_Y_TILING(dev))
2291                 tile_width = 128;
2292         else
2293                 tile_width = 512;
2294
2295         /* Note: pitch better be a power of two tile widths */
2296         pitch_val = obj_priv->stride / tile_width;
2297         pitch_val = ffs(pitch_val) - 1;
2298
2299         if (obj_priv->tiling_mode == I915_TILING_Y &&
2300             HAS_128_BYTE_Y_TILING(dev))
2301                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2302         else
2303                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2304
2305         val = obj_priv->gtt_offset;
2306         if (obj_priv->tiling_mode == I915_TILING_Y)
2307                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2308         val |= I915_FENCE_SIZE_BITS(obj->size);
2309         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2310         val |= I830_FENCE_REG_VALID;
2311
2312         if (regnum < 8)
2313                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2314         else
2315                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2316         I915_WRITE(fence_reg, val);
2317 }
2318
2319 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2320 {
2321         struct drm_gem_object *obj = reg->obj;
2322         struct drm_device *dev = obj->dev;
2323         drm_i915_private_t *dev_priv = dev->dev_private;
2324         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2325         int regnum = obj_priv->fence_reg;
2326         uint32_t val;
2327         uint32_t pitch_val;
2328         uint32_t fence_size_bits;
2329
2330         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2331             (obj_priv->gtt_offset & (obj->size - 1))) {
2332                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2333                      __func__, obj_priv->gtt_offset);
2334                 return;
2335         }
2336
2337         pitch_val = obj_priv->stride / 128;
2338         pitch_val = ffs(pitch_val) - 1;
2339         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2340
2341         val = obj_priv->gtt_offset;
2342         if (obj_priv->tiling_mode == I915_TILING_Y)
2343                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2344         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2345         WARN_ON(fence_size_bits & ~0x00000f00);
2346         val |= fence_size_bits;
2347         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2348         val |= I830_FENCE_REG_VALID;
2349
2350         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2351 }
2352
2353 static int i915_find_fence_reg(struct drm_device *dev,
2354                                bool interruptible)
2355 {
2356         struct drm_i915_fence_reg *reg = NULL;
2357         struct drm_i915_gem_object *obj_priv = NULL;
2358         struct drm_i915_private *dev_priv = dev->dev_private;
2359         struct drm_gem_object *obj = NULL;
2360         int i, avail, ret;
2361
2362         /* First try to find a free reg */
2363         avail = 0;
2364         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2365                 reg = &dev_priv->fence_regs[i];
2366                 if (!reg->obj)
2367                         return i;
2368
2369                 obj_priv = to_intel_bo(reg->obj);
2370                 if (!obj_priv->pin_count)
2371                     avail++;
2372         }
2373
2374         if (avail == 0)
2375                 return -ENOSPC;
2376
2377         /* None available, try to steal one or wait for a user to finish */
2378         i = I915_FENCE_REG_NONE;
2379         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2380                             lru_list) {
2381                 obj = reg->obj;
2382                 obj_priv = to_intel_bo(obj);
2383
2384                 if (obj_priv->pin_count)
2385                         continue;
2386
2387                 /* found one! */
2388                 i = obj_priv->fence_reg;
2389                 break;
2390         }
2391
2392         BUG_ON(i == I915_FENCE_REG_NONE);
2393
2394         /* We only have a reference on obj from the active list. put_fence_reg
2395          * might drop that one, causing a use-after-free in it. So hold a
2396          * private reference to obj like the other callers of put_fence_reg
2397          * (set_tiling ioctl) do. */
2398         drm_gem_object_reference(obj);
2399         ret = i915_gem_object_put_fence_reg(obj, interruptible);
2400         drm_gem_object_unreference(obj);
2401         if (ret != 0)
2402                 return ret;
2403
2404         return i;
2405 }
2406
2407 /**
2408  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2409  * @obj: object to map through a fence reg
2410  *
2411  * When mapping objects through the GTT, userspace wants to be able to write
2412  * to them without having to worry about swizzling if the object is tiled.
2413  *
2414  * This function walks the fence regs looking for a free one for @obj,
2415  * stealing one if it can't find any.
2416  *
2417  * It then sets up the reg based on the object's properties: address, pitch
2418  * and tiling format.
2419  */
2420 int
2421 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2422                               bool interruptible)
2423 {
2424         struct drm_device *dev = obj->dev;
2425         struct drm_i915_private *dev_priv = dev->dev_private;
2426         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2427         struct drm_i915_fence_reg *reg = NULL;
2428         int ret;
2429
2430         /* Just update our place in the LRU if our fence is getting used. */
2431         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2432                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2433                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2434                 return 0;
2435         }
2436
2437         switch (obj_priv->tiling_mode) {
2438         case I915_TILING_NONE:
2439                 WARN(1, "allocating a fence for non-tiled object?\n");
2440                 break;
2441         case I915_TILING_X:
2442                 if (!obj_priv->stride)
2443                         return -EINVAL;
2444                 WARN((obj_priv->stride & (512 - 1)),
2445                      "object 0x%08x is X tiled but has non-512B pitch\n",
2446                      obj_priv->gtt_offset);
2447                 break;
2448         case I915_TILING_Y:
2449                 if (!obj_priv->stride)
2450                         return -EINVAL;
2451                 WARN((obj_priv->stride & (128 - 1)),
2452                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2453                      obj_priv->gtt_offset);
2454                 break;
2455         }
2456
2457         ret = i915_find_fence_reg(dev, interruptible);
2458         if (ret < 0)
2459                 return ret;
2460
2461         obj_priv->fence_reg = ret;
2462         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2463         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2464
2465         reg->obj = obj;
2466
2467         switch (INTEL_INFO(dev)->gen) {
2468         case 6:
2469                 sandybridge_write_fence_reg(reg);
2470                 break;
2471         case 5:
2472         case 4:
2473                 i965_write_fence_reg(reg);
2474                 break;
2475         case 3:
2476                 i915_write_fence_reg(reg);
2477                 break;
2478         case 2:
2479                 i830_write_fence_reg(reg);
2480                 break;
2481         }
2482
2483         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2484                         obj_priv->tiling_mode);
2485
2486         return 0;
2487 }
2488
2489 /**
2490  * i915_gem_clear_fence_reg - clear out fence register info
2491  * @obj: object to clear
2492  *
2493  * Zeroes out the fence register itself and clears out the associated
2494  * data structures in dev_priv and obj_priv.
2495  */
2496 static void
2497 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2498 {
2499         struct drm_device *dev = obj->dev;
2500         drm_i915_private_t *dev_priv = dev->dev_private;
2501         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2502         struct drm_i915_fence_reg *reg =
2503                 &dev_priv->fence_regs[obj_priv->fence_reg];
2504         uint32_t fence_reg;
2505
2506         switch (INTEL_INFO(dev)->gen) {
2507         case 6:
2508                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2509                              (obj_priv->fence_reg * 8), 0);
2510                 break;
2511         case 5:
2512         case 4:
2513                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2514                 break;
2515         case 3:
2516                 if (obj_priv->fence_reg > 8)
2517                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2518                 else
2519         case 2:
2520                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2521
2522                 I915_WRITE(fence_reg, 0);
2523                 break;
2524         }
2525
2526         reg->obj = NULL;
2527         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2528         list_del_init(&reg->lru_list);
2529 }
2530
2531 /**
2532  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2533  * to the buffer to finish, and then resets the fence register.
2534  * @obj: tiled object holding a fence register.
2535  * @bool: whether the wait upon the fence is interruptible
2536  *
2537  * Zeroes out the fence register itself and clears out the associated
2538  * data structures in dev_priv and obj_priv.
2539  */
2540 int
2541 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2542                               bool interruptible)
2543 {
2544         struct drm_device *dev = obj->dev;
2545         struct drm_i915_private *dev_priv = dev->dev_private;
2546         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2547         struct drm_i915_fence_reg *reg;
2548
2549         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2550                 return 0;
2551
2552         /* If we've changed tiling, GTT-mappings of the object
2553          * need to re-fault to ensure that the correct fence register
2554          * setup is in place.
2555          */
2556         i915_gem_release_mmap(obj);
2557
2558         /* On the i915, GPU access to tiled buffers is via a fence,
2559          * therefore we must wait for any outstanding access to complete
2560          * before clearing the fence.
2561          */
2562         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2563         if (reg->gpu) {
2564                 int ret;
2565
2566                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2567                 if (ret)
2568                         return ret;
2569
2570                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2571                 if (ret)
2572                         return ret;
2573
2574                 reg->gpu = false;
2575         }
2576
2577         i915_gem_object_flush_gtt_write_domain(obj);
2578         i915_gem_clear_fence_reg(obj);
2579
2580         return 0;
2581 }
2582
2583 /**
2584  * Finds free space in the GTT aperture and binds the object there.
2585  */
2586 static int
2587 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2588 {
2589         struct drm_device *dev = obj->dev;
2590         drm_i915_private_t *dev_priv = dev->dev_private;
2591         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2592         struct drm_mm_node *free_space;
2593         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2594         int ret;
2595
2596         if (obj_priv->madv != I915_MADV_WILLNEED) {
2597                 DRM_ERROR("Attempting to bind a purgeable object\n");
2598                 return -EINVAL;
2599         }
2600
2601         if (alignment == 0)
2602                 alignment = i915_gem_get_gtt_alignment(obj);
2603         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2604                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2605                 return -EINVAL;
2606         }
2607
2608         /* If the object is bigger than the entire aperture, reject it early
2609          * before evicting everything in a vain attempt to find space.
2610          */
2611         if (obj->size > dev->gtt_total) {
2612                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2613                 return -E2BIG;
2614         }
2615
2616  search_free:
2617         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2618                                         obj->size, alignment, 0);
2619         if (free_space != NULL) {
2620                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2621                                                        alignment);
2622                 if (obj_priv->gtt_space != NULL)
2623                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2624         }
2625         if (obj_priv->gtt_space == NULL) {
2626                 /* If the gtt is empty and we're still having trouble
2627                  * fitting our object in, we're out of memory.
2628                  */
2629 #if WATCH_LRU
2630                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2631 #endif
2632                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2633                 if (ret)
2634                         return ret;
2635
2636                 goto search_free;
2637         }
2638
2639 #if WATCH_BUF
2640         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2641                  obj->size, obj_priv->gtt_offset);
2642 #endif
2643         ret = i915_gem_object_get_pages(obj, gfpmask);
2644         if (ret) {
2645                 drm_mm_put_block(obj_priv->gtt_space);
2646                 obj_priv->gtt_space = NULL;
2647
2648                 if (ret == -ENOMEM) {
2649                         /* first try to clear up some space from the GTT */
2650                         ret = i915_gem_evict_something(dev, obj->size,
2651                                                        alignment);
2652                         if (ret) {
2653                                 /* now try to shrink everyone else */
2654                                 if (gfpmask) {
2655                                         gfpmask = 0;
2656                                         goto search_free;
2657                                 }
2658
2659                                 return ret;
2660                         }
2661
2662                         goto search_free;
2663                 }
2664
2665                 return ret;
2666         }
2667
2668         /* Create an AGP memory structure pointing at our pages, and bind it
2669          * into the GTT.
2670          */
2671         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2672                                                obj_priv->pages,
2673                                                obj->size >> PAGE_SHIFT,
2674                                                obj_priv->gtt_offset,
2675                                                obj_priv->agp_type);
2676         if (obj_priv->agp_mem == NULL) {
2677                 i915_gem_object_put_pages(obj);
2678                 drm_mm_put_block(obj_priv->gtt_space);
2679                 obj_priv->gtt_space = NULL;
2680
2681                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2682                 if (ret)
2683                         return ret;
2684
2685                 goto search_free;
2686         }
2687         atomic_inc(&dev->gtt_count);
2688         atomic_add(obj->size, &dev->gtt_memory);
2689
2690         /* keep track of bounds object by adding it to the inactive list */
2691         list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2692
2693         /* Assert that the object is not currently in any GPU domain. As it
2694          * wasn't in the GTT, there shouldn't be any way it could have been in
2695          * a GPU cache
2696          */
2697         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2698         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2699
2700         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2701
2702         return 0;
2703 }
2704
2705 void
2706 i915_gem_clflush_object(struct drm_gem_object *obj)
2707 {
2708         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2709
2710         /* If we don't have a page list set up, then we're not pinned
2711          * to GPU, and we can ignore the cache flush because it'll happen
2712          * again at bind time.
2713          */
2714         if (obj_priv->pages == NULL)
2715                 return;
2716
2717         trace_i915_gem_object_clflush(obj);
2718
2719         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2720 }
2721
2722 /** Flushes any GPU write domain for the object if it's dirty. */
2723 static int
2724 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2725                                        bool pipelined)
2726 {
2727         struct drm_device *dev = obj->dev;
2728         uint32_t old_write_domain;
2729
2730         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2731                 return 0;
2732
2733         /* Queue the GPU write cache flushing we need. */
2734         old_write_domain = obj->write_domain;
2735         i915_gem_flush_ring(dev, NULL,
2736                             to_intel_bo(obj)->ring,
2737                             0, obj->write_domain);
2738         BUG_ON(obj->write_domain);
2739
2740         trace_i915_gem_object_change_domain(obj,
2741                                             obj->read_domains,
2742                                             old_write_domain);
2743
2744         if (pipelined)
2745                 return 0;
2746
2747         return i915_gem_object_wait_rendering(obj, true);
2748 }
2749
2750 /** Flushes the GTT write domain for the object if it's dirty. */
2751 static void
2752 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2753 {
2754         uint32_t old_write_domain;
2755
2756         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2757                 return;
2758
2759         /* No actual flushing is required for the GTT write domain.   Writes
2760          * to it immediately go to main memory as far as we know, so there's
2761          * no chipset flush.  It also doesn't land in render cache.
2762          */
2763         old_write_domain = obj->write_domain;
2764         obj->write_domain = 0;
2765
2766         trace_i915_gem_object_change_domain(obj,
2767                                             obj->read_domains,
2768                                             old_write_domain);
2769 }
2770
2771 /** Flushes the CPU write domain for the object if it's dirty. */
2772 static void
2773 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2774 {
2775         struct drm_device *dev = obj->dev;
2776         uint32_t old_write_domain;
2777
2778         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2779                 return;
2780
2781         i915_gem_clflush_object(obj);
2782         drm_agp_chipset_flush(dev);
2783         old_write_domain = obj->write_domain;
2784         obj->write_domain = 0;
2785
2786         trace_i915_gem_object_change_domain(obj,
2787                                             obj->read_domains,
2788                                             old_write_domain);
2789 }
2790
2791 /**
2792  * Moves a single object to the GTT read, and possibly write domain.
2793  *
2794  * This function returns when the move is complete, including waiting on
2795  * flushes to occur.
2796  */
2797 int
2798 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2799 {
2800         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2801         uint32_t old_write_domain, old_read_domains;
2802         int ret;
2803
2804         /* Not valid to be called on unbound objects. */
2805         if (obj_priv->gtt_space == NULL)
2806                 return -EINVAL;
2807
2808         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2809         if (ret != 0)
2810                 return ret;
2811
2812         i915_gem_object_flush_cpu_write_domain(obj);
2813
2814         if (write) {
2815                 ret = i915_gem_object_wait_rendering(obj, true);
2816                 if (ret)
2817                         return ret;
2818         }
2819
2820         old_write_domain = obj->write_domain;
2821         old_read_domains = obj->read_domains;
2822
2823         /* It should now be out of any other write domains, and we can update
2824          * the domain values for our changes.
2825          */
2826         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2827         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2828         if (write) {
2829                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2830                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2831                 obj_priv->dirty = 1;
2832         }
2833
2834         trace_i915_gem_object_change_domain(obj,
2835                                             old_read_domains,
2836                                             old_write_domain);
2837
2838         return 0;
2839 }
2840
2841 /*
2842  * Prepare buffer for display plane. Use uninterruptible for possible flush
2843  * wait, as in modesetting process we're not supposed to be interrupted.
2844  */
2845 int
2846 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2847                                      bool pipelined)
2848 {
2849         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2850         uint32_t old_read_domains;
2851         int ret;
2852
2853         /* Not valid to be called on unbound objects. */
2854         if (obj_priv->gtt_space == NULL)
2855                 return -EINVAL;
2856
2857         ret = i915_gem_object_flush_gpu_write_domain(obj, pipelined);
2858         if (ret)
2859                 return ret;
2860
2861         i915_gem_object_flush_cpu_write_domain(obj);
2862
2863         old_read_domains = obj->read_domains;
2864         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2865
2866         trace_i915_gem_object_change_domain(obj,
2867                                             old_read_domains,
2868                                             obj->write_domain);
2869
2870         return 0;
2871 }
2872
2873 /**
2874  * Moves a single object to the CPU read, and possibly write domain.
2875  *
2876  * This function returns when the move is complete, including waiting on
2877  * flushes to occur.
2878  */
2879 static int
2880 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2881 {
2882         uint32_t old_write_domain, old_read_domains;
2883         int ret;
2884
2885         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2886         if (ret != 0)
2887                 return ret;
2888
2889         i915_gem_object_flush_gtt_write_domain(obj);
2890
2891         /* If we have a partially-valid cache of the object in the CPU,
2892          * finish invalidating it and free the per-page flags.
2893          */
2894         i915_gem_object_set_to_full_cpu_read_domain(obj);
2895
2896         if (write) {
2897                 ret = i915_gem_object_wait_rendering(obj, true);
2898                 if (ret)
2899                         return ret;
2900         }
2901
2902         old_write_domain = obj->write_domain;
2903         old_read_domains = obj->read_domains;
2904
2905         /* Flush the CPU cache if it's still invalid. */
2906         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2907                 i915_gem_clflush_object(obj);
2908
2909                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2910         }
2911
2912         /* It should now be out of any other write domains, and we can update
2913          * the domain values for our changes.
2914          */
2915         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2916
2917         /* If we're writing through the CPU, then the GPU read domains will
2918          * need to be invalidated at next use.
2919          */
2920         if (write) {
2921                 obj->read_domains = I915_GEM_DOMAIN_CPU;
2922                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2923         }
2924
2925         trace_i915_gem_object_change_domain(obj,
2926                                             old_read_domains,
2927                                             old_write_domain);
2928
2929         return 0;
2930 }
2931
2932 /*
2933  * Set the next domain for the specified object. This
2934  * may not actually perform the necessary flushing/invaliding though,
2935  * as that may want to be batched with other set_domain operations
2936  *
2937  * This is (we hope) the only really tricky part of gem. The goal
2938  * is fairly simple -- track which caches hold bits of the object
2939  * and make sure they remain coherent. A few concrete examples may
2940  * help to explain how it works. For shorthand, we use the notation
2941  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2942  * a pair of read and write domain masks.
2943  *
2944  * Case 1: the batch buffer
2945  *
2946  *      1. Allocated
2947  *      2. Written by CPU
2948  *      3. Mapped to GTT
2949  *      4. Read by GPU
2950  *      5. Unmapped from GTT
2951  *      6. Freed
2952  *
2953  *      Let's take these a step at a time
2954  *
2955  *      1. Allocated
2956  *              Pages allocated from the kernel may still have
2957  *              cache contents, so we set them to (CPU, CPU) always.
2958  *      2. Written by CPU (using pwrite)
2959  *              The pwrite function calls set_domain (CPU, CPU) and
2960  *              this function does nothing (as nothing changes)
2961  *      3. Mapped by GTT
2962  *              This function asserts that the object is not
2963  *              currently in any GPU-based read or write domains
2964  *      4. Read by GPU
2965  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2966  *              As write_domain is zero, this function adds in the
2967  *              current read domains (CPU+COMMAND, 0).
2968  *              flush_domains is set to CPU.
2969  *              invalidate_domains is set to COMMAND
2970  *              clflush is run to get data out of the CPU caches
2971  *              then i915_dev_set_domain calls i915_gem_flush to
2972  *              emit an MI_FLUSH and drm_agp_chipset_flush
2973  *      5. Unmapped from GTT
2974  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2975  *              flush_domains and invalidate_domains end up both zero
2976  *              so no flushing/invalidating happens
2977  *      6. Freed
2978  *              yay, done
2979  *
2980  * Case 2: The shared render buffer
2981  *
2982  *      1. Allocated
2983  *      2. Mapped to GTT
2984  *      3. Read/written by GPU
2985  *      4. set_domain to (CPU,CPU)
2986  *      5. Read/written by CPU
2987  *      6. Read/written by GPU
2988  *
2989  *      1. Allocated
2990  *              Same as last example, (CPU, CPU)
2991  *      2. Mapped to GTT
2992  *              Nothing changes (assertions find that it is not in the GPU)
2993  *      3. Read/written by GPU
2994  *              execbuffer calls set_domain (RENDER, RENDER)
2995  *              flush_domains gets CPU
2996  *              invalidate_domains gets GPU
2997  *              clflush (obj)
2998  *              MI_FLUSH and drm_agp_chipset_flush
2999  *      4. set_domain (CPU, CPU)
3000  *              flush_domains gets GPU
3001  *              invalidate_domains gets CPU
3002  *              wait_rendering (obj) to make sure all drawing is complete.
3003  *              This will include an MI_FLUSH to get the data from GPU
3004  *              to memory
3005  *              clflush (obj) to invalidate the CPU cache
3006  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3007  *      5. Read/written by CPU
3008  *              cache lines are loaded and dirtied
3009  *      6. Read written by GPU
3010  *              Same as last GPU access
3011  *
3012  * Case 3: The constant buffer
3013  *
3014  *      1. Allocated
3015  *      2. Written by CPU
3016  *      3. Read by GPU
3017  *      4. Updated (written) by CPU again
3018  *      5. Read by GPU
3019  *
3020  *      1. Allocated
3021  *              (CPU, CPU)
3022  *      2. Written by CPU
3023  *              (CPU, CPU)
3024  *      3. Read by GPU
3025  *              (CPU+RENDER, 0)
3026  *              flush_domains = CPU
3027  *              invalidate_domains = RENDER
3028  *              clflush (obj)
3029  *              MI_FLUSH
3030  *              drm_agp_chipset_flush
3031  *      4. Updated (written) by CPU again
3032  *              (CPU, CPU)
3033  *              flush_domains = 0 (no previous write domain)
3034  *              invalidate_domains = 0 (no new read domains)
3035  *      5. Read by GPU
3036  *              (CPU+RENDER, 0)
3037  *              flush_domains = CPU
3038  *              invalidate_domains = RENDER
3039  *              clflush (obj)
3040  *              MI_FLUSH
3041  *              drm_agp_chipset_flush
3042  */
3043 static void
3044 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3045 {
3046         struct drm_device               *dev = obj->dev;
3047         struct drm_i915_private         *dev_priv = dev->dev_private;
3048         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3049         uint32_t                        invalidate_domains = 0;
3050         uint32_t                        flush_domains = 0;
3051         uint32_t                        old_read_domains;
3052
3053         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3054         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
3055
3056         intel_mark_busy(dev, obj);
3057
3058 #if WATCH_BUF
3059         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3060                  __func__, obj,
3061                  obj->read_domains, obj->pending_read_domains,
3062                  obj->write_domain, obj->pending_write_domain);
3063 #endif
3064         /*
3065          * If the object isn't moving to a new write domain,
3066          * let the object stay in multiple read domains
3067          */
3068         if (obj->pending_write_domain == 0)
3069                 obj->pending_read_domains |= obj->read_domains;
3070         else
3071                 obj_priv->dirty = 1;
3072
3073         /*
3074          * Flush the current write domain if
3075          * the new read domains don't match. Invalidate
3076          * any read domains which differ from the old
3077          * write domain
3078          */
3079         if (obj->write_domain &&
3080             obj->write_domain != obj->pending_read_domains) {
3081                 flush_domains |= obj->write_domain;
3082                 invalidate_domains |=
3083                         obj->pending_read_domains & ~obj->write_domain;
3084         }
3085         /*
3086          * Invalidate any read caches which may have
3087          * stale data. That is, any new read domains.
3088          */
3089         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3090         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3091 #if WATCH_BUF
3092                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3093                          __func__, flush_domains, invalidate_domains);
3094 #endif
3095                 i915_gem_clflush_object(obj);
3096         }
3097
3098         old_read_domains = obj->read_domains;
3099
3100         /* The actual obj->write_domain will be updated with
3101          * pending_write_domain after we emit the accumulated flush for all
3102          * of our domain changes in execbuffers (which clears objects'
3103          * write_domains).  So if we have a current write domain that we
3104          * aren't changing, set pending_write_domain to that.
3105          */
3106         if (flush_domains == 0 && obj->pending_write_domain == 0)
3107                 obj->pending_write_domain = obj->write_domain;
3108         obj->read_domains = obj->pending_read_domains;
3109
3110         dev->invalidate_domains |= invalidate_domains;
3111         dev->flush_domains |= flush_domains;
3112         if (obj_priv->ring)
3113                 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3114 #if WATCH_BUF
3115         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3116                  __func__,
3117                  obj->read_domains, obj->write_domain,
3118                  dev->invalidate_domains, dev->flush_domains);
3119 #endif
3120
3121         trace_i915_gem_object_change_domain(obj,
3122                                             old_read_domains,
3123                                             obj->write_domain);
3124 }
3125
3126 /**
3127  * Moves the object from a partially CPU read to a full one.
3128  *
3129  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3130  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3131  */
3132 static void
3133 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3134 {
3135         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3136
3137         if (!obj_priv->page_cpu_valid)
3138                 return;
3139
3140         /* If we're partially in the CPU read domain, finish moving it in.
3141          */
3142         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3143                 int i;
3144
3145                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3146                         if (obj_priv->page_cpu_valid[i])
3147                                 continue;
3148                         drm_clflush_pages(obj_priv->pages + i, 1);
3149                 }
3150         }
3151
3152         /* Free the page_cpu_valid mappings which are now stale, whether
3153          * or not we've got I915_GEM_DOMAIN_CPU.
3154          */
3155         kfree(obj_priv->page_cpu_valid);
3156         obj_priv->page_cpu_valid = NULL;
3157 }
3158
3159 /**
3160  * Set the CPU read domain on a range of the object.
3161  *
3162  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3163  * not entirely valid.  The page_cpu_valid member of the object flags which
3164  * pages have been flushed, and will be respected by
3165  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3166  * of the whole object.
3167  *
3168  * This function returns when the move is complete, including waiting on
3169  * flushes to occur.
3170  */
3171 static int
3172 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3173                                           uint64_t offset, uint64_t size)
3174 {
3175         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3176         uint32_t old_read_domains;
3177         int i, ret;
3178
3179         if (offset == 0 && size == obj->size)
3180                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3181
3182         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3183         if (ret != 0)
3184                 return ret;
3185         i915_gem_object_flush_gtt_write_domain(obj);
3186
3187         /* If we're already fully in the CPU read domain, we're done. */
3188         if (obj_priv->page_cpu_valid == NULL &&
3189             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3190                 return 0;
3191
3192         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3193          * newly adding I915_GEM_DOMAIN_CPU
3194          */
3195         if (obj_priv->page_cpu_valid == NULL) {
3196                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3197                                                    GFP_KERNEL);
3198                 if (obj_priv->page_cpu_valid == NULL)
3199                         return -ENOMEM;
3200         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3201                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3202
3203         /* Flush the cache on any pages that are still invalid from the CPU's
3204          * perspective.
3205          */
3206         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3207              i++) {
3208                 if (obj_priv->page_cpu_valid[i])
3209                         continue;
3210
3211                 drm_clflush_pages(obj_priv->pages + i, 1);
3212
3213                 obj_priv->page_cpu_valid[i] = 1;
3214         }
3215
3216         /* It should now be out of any other write domains, and we can update
3217          * the domain values for our changes.
3218          */
3219         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3220
3221         old_read_domains = obj->read_domains;
3222         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3223
3224         trace_i915_gem_object_change_domain(obj,
3225                                             old_read_domains,
3226                                             obj->write_domain);
3227
3228         return 0;
3229 }
3230
3231 /**
3232  * Pin an object to the GTT and evaluate the relocations landing in it.
3233  */
3234 static int
3235 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3236                                  struct drm_file *file_priv,
3237                                  struct drm_i915_gem_exec_object2 *entry,
3238                                  struct drm_i915_gem_relocation_entry *relocs)
3239 {
3240         struct drm_device *dev = obj->dev;
3241         drm_i915_private_t *dev_priv = dev->dev_private;
3242         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3243         int i, ret;
3244         void __iomem *reloc_page;
3245         bool need_fence;
3246
3247         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3248                      obj_priv->tiling_mode != I915_TILING_NONE;
3249
3250         /* Check fence reg constraints and rebind if necessary */
3251         if (need_fence &&
3252             !i915_gem_object_fence_offset_ok(obj,
3253                                              obj_priv->tiling_mode)) {
3254                 ret = i915_gem_object_unbind(obj);
3255                 if (ret)
3256                         return ret;
3257         }
3258
3259         /* Choose the GTT offset for our buffer and put it there. */
3260         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3261         if (ret)
3262                 return ret;
3263
3264         /*
3265          * Pre-965 chips need a fence register set up in order to
3266          * properly handle blits to/from tiled surfaces.
3267          */
3268         if (need_fence) {
3269                 ret = i915_gem_object_get_fence_reg(obj, true);
3270                 if (ret != 0) {
3271                         i915_gem_object_unpin(obj);
3272                         return ret;
3273                 }
3274
3275                 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true;
3276         }
3277
3278         entry->offset = obj_priv->gtt_offset;
3279
3280         /* Apply the relocations, using the GTT aperture to avoid cache
3281          * flushing requirements.
3282          */
3283         for (i = 0; i < entry->relocation_count; i++) {
3284                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3285                 struct drm_gem_object *target_obj;
3286                 struct drm_i915_gem_object *target_obj_priv;
3287                 uint32_t reloc_val, reloc_offset;
3288                 uint32_t __iomem *reloc_entry;
3289
3290                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3291                                                    reloc->target_handle);
3292                 if (target_obj == NULL) {
3293                         i915_gem_object_unpin(obj);
3294                         return -ENOENT;
3295                 }
3296                 target_obj_priv = to_intel_bo(target_obj);
3297
3298 #if WATCH_RELOC
3299                 DRM_INFO("%s: obj %p offset %08x target %d "
3300                          "read %08x write %08x gtt %08x "
3301                          "presumed %08x delta %08x\n",
3302                          __func__,
3303                          obj,
3304                          (int) reloc->offset,
3305                          (int) reloc->target_handle,
3306                          (int) reloc->read_domains,
3307                          (int) reloc->write_domain,
3308                          (int) target_obj_priv->gtt_offset,
3309                          (int) reloc->presumed_offset,
3310                          reloc->delta);
3311 #endif
3312
3313                 /* The target buffer should have appeared before us in the
3314                  * exec_object list, so it should have a GTT space bound by now.
3315                  */
3316                 if (target_obj_priv->gtt_space == NULL) {
3317                         DRM_ERROR("No GTT space found for object %d\n",
3318                                   reloc->target_handle);
3319                         drm_gem_object_unreference(target_obj);
3320                         i915_gem_object_unpin(obj);
3321                         return -EINVAL;
3322                 }
3323
3324                 /* Validate that the target is in a valid r/w GPU domain */
3325                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3326                         DRM_ERROR("reloc with multiple write domains: "
3327                                   "obj %p target %d offset %d "
3328                                   "read %08x write %08x",
3329                                   obj, reloc->target_handle,
3330                                   (int) reloc->offset,
3331                                   reloc->read_domains,
3332                                   reloc->write_domain);
3333                         return -EINVAL;
3334                 }
3335                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3336                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3337                         DRM_ERROR("reloc with read/write CPU domains: "
3338                                   "obj %p target %d offset %d "
3339                                   "read %08x write %08x",
3340                                   obj, reloc->target_handle,
3341                                   (int) reloc->offset,
3342                                   reloc->read_domains,
3343                                   reloc->write_domain);
3344                         drm_gem_object_unreference(target_obj);
3345                         i915_gem_object_unpin(obj);
3346                         return -EINVAL;
3347                 }
3348                 if (reloc->write_domain && target_obj->pending_write_domain &&
3349                     reloc->write_domain != target_obj->pending_write_domain) {
3350                         DRM_ERROR("Write domain conflict: "
3351                                   "obj %p target %d offset %d "
3352                                   "new %08x old %08x\n",
3353                                   obj, reloc->target_handle,
3354                                   (int) reloc->offset,
3355                                   reloc->write_domain,
3356                                   target_obj->pending_write_domain);
3357                         drm_gem_object_unreference(target_obj);
3358                         i915_gem_object_unpin(obj);
3359                         return -EINVAL;
3360                 }
3361
3362                 target_obj->pending_read_domains |= reloc->read_domains;
3363                 target_obj->pending_write_domain |= reloc->write_domain;
3364
3365                 /* If the relocation already has the right value in it, no
3366                  * more work needs to be done.
3367                  */
3368                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3369                         drm_gem_object_unreference(target_obj);
3370                         continue;
3371                 }
3372
3373                 /* Check that the relocation address is valid... */
3374                 if (reloc->offset > obj->size - 4) {
3375                         DRM_ERROR("Relocation beyond object bounds: "
3376                                   "obj %p target %d offset %d size %d.\n",
3377                                   obj, reloc->target_handle,
3378                                   (int) reloc->offset, (int) obj->size);
3379                         drm_gem_object_unreference(target_obj);
3380                         i915_gem_object_unpin(obj);
3381                         return -EINVAL;
3382                 }
3383                 if (reloc->offset & 3) {
3384                         DRM_ERROR("Relocation not 4-byte aligned: "
3385                                   "obj %p target %d offset %d.\n",
3386                                   obj, reloc->target_handle,
3387                                   (int) reloc->offset);
3388                         drm_gem_object_unreference(target_obj);
3389                         i915_gem_object_unpin(obj);
3390                         return -EINVAL;
3391                 }
3392
3393                 /* and points to somewhere within the target object. */
3394                 if (reloc->delta >= target_obj->size) {
3395                         DRM_ERROR("Relocation beyond target object bounds: "
3396                                   "obj %p target %d delta %d size %d.\n",
3397                                   obj, reloc->target_handle,
3398                                   (int) reloc->delta, (int) target_obj->size);
3399                         drm_gem_object_unreference(target_obj);
3400                         i915_gem_object_unpin(obj);
3401                         return -EINVAL;
3402                 }
3403
3404                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3405                 if (ret != 0) {
3406                         drm_gem_object_unreference(target_obj);
3407                         i915_gem_object_unpin(obj);
3408                         return -EINVAL;
3409                 }
3410
3411                 /* Map the page containing the relocation we're going to
3412                  * perform.
3413                  */
3414                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3415                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3416                                                       (reloc_offset &
3417                                                        ~(PAGE_SIZE - 1)),
3418                                                       KM_USER0);
3419                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3420                                                    (reloc_offset & (PAGE_SIZE - 1)));
3421                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3422
3423 #if WATCH_BUF
3424                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3425                           obj, (unsigned int) reloc->offset,
3426                           readl(reloc_entry), reloc_val);
3427 #endif
3428                 writel(reloc_val, reloc_entry);
3429                 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3430
3431                 /* The updated presumed offset for this entry will be
3432                  * copied back out to the user.
3433                  */
3434                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3435
3436                 drm_gem_object_unreference(target_obj);
3437         }
3438
3439 #if WATCH_BUF
3440         if (0)
3441                 i915_gem_dump_object(obj, 128, __func__, ~0);
3442 #endif
3443         return 0;
3444 }
3445
3446 /* Throttle our rendering by waiting until the ring has completed our requests
3447  * emitted over 20 msec ago.
3448  *
3449  * Note that if we were to use the current jiffies each time around the loop,
3450  * we wouldn't escape the function with any frames outstanding if the time to
3451  * render a frame was over 20ms.
3452  *
3453  * This should get us reasonable parallelism between CPU and GPU but also
3454  * relatively low latency when blocking on a particular request to finish.
3455  */
3456 static int
3457 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3458 {
3459         struct drm_i915_private *dev_priv = dev->dev_private;
3460         struct drm_i915_file_private *file_priv = file->driver_priv;
3461         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3462         struct drm_i915_gem_request *request;
3463         struct intel_ring_buffer *ring = NULL;
3464         u32 seqno = 0;
3465         int ret;
3466
3467         mutex_lock(&file_priv->mutex);
3468         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3469                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3470                         break;
3471
3472                 ring = request->ring;
3473                 seqno = request->seqno;
3474         }
3475         mutex_unlock(&file_priv->mutex);
3476
3477         if (seqno == 0)
3478                 return 0;
3479
3480         ret = 0;
3481         if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3482                 /* And wait for the seqno passing without holding any locks and
3483                  * causing extra latency for others. This is safe as the irq
3484                  * generation is designed to be run atomically and so is
3485                  * lockless.
3486                  */
3487                 ring->user_irq_get(dev, ring);
3488                 ret = wait_event_interruptible(ring->irq_queue,
3489                                                i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3490                                                || atomic_read(&dev_priv->mm.wedged));
3491                 ring->user_irq_put(dev, ring);
3492
3493                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3494                         ret = -EIO;
3495         }
3496
3497         if (ret == 0)
3498                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3499
3500         return ret;
3501 }
3502
3503 static int
3504 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3505                               uint32_t buffer_count,
3506                               struct drm_i915_gem_relocation_entry **relocs)
3507 {
3508         uint32_t reloc_count = 0, reloc_index = 0, i;
3509         int ret;
3510
3511         *relocs = NULL;
3512         for (i = 0; i < buffer_count; i++) {
3513                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3514                         return -EINVAL;
3515                 reloc_count += exec_list[i].relocation_count;
3516         }
3517
3518         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3519         if (*relocs == NULL) {
3520                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3521                 return -ENOMEM;
3522         }
3523
3524         for (i = 0; i < buffer_count; i++) {
3525                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3526
3527                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3528
3529                 ret = copy_from_user(&(*relocs)[reloc_index],
3530                                      user_relocs,
3531                                      exec_list[i].relocation_count *
3532                                      sizeof(**relocs));
3533                 if (ret != 0) {
3534                         drm_free_large(*relocs);
3535                         *relocs = NULL;
3536                         return -EFAULT;
3537                 }
3538
3539                 reloc_index += exec_list[i].relocation_count;
3540         }
3541
3542         return 0;
3543 }
3544
3545 static int
3546 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3547                             uint32_t buffer_count,
3548                             struct drm_i915_gem_relocation_entry *relocs)
3549 {
3550         uint32_t reloc_count = 0, i;
3551         int ret = 0;
3552
3553         if (relocs == NULL)
3554             return 0;
3555
3556         for (i = 0; i < buffer_count; i++) {
3557                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3558                 int unwritten;
3559
3560                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3561
3562                 unwritten = copy_to_user(user_relocs,
3563                                          &relocs[reloc_count],
3564                                          exec_list[i].relocation_count *
3565                                          sizeof(*relocs));
3566
3567                 if (unwritten) {
3568                         ret = -EFAULT;
3569                         goto err;
3570                 }
3571
3572                 reloc_count += exec_list[i].relocation_count;
3573         }
3574
3575 err:
3576         drm_free_large(relocs);
3577
3578         return ret;
3579 }
3580
3581 static int
3582 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3583                            uint64_t exec_offset)
3584 {
3585         uint32_t exec_start, exec_len;
3586
3587         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3588         exec_len = (uint32_t) exec->batch_len;
3589
3590         if ((exec_start | exec_len) & 0x7)
3591                 return -EINVAL;
3592
3593         if (!exec_start)
3594                 return -EINVAL;
3595
3596         return 0;
3597 }
3598
3599 static int
3600 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3601                                struct drm_gem_object **object_list,
3602                                int count)
3603 {
3604         drm_i915_private_t *dev_priv = dev->dev_private;
3605         struct drm_i915_gem_object *obj_priv;
3606         DEFINE_WAIT(wait);
3607         int i, ret = 0;
3608
3609         for (;;) {
3610                 prepare_to_wait(&dev_priv->pending_flip_queue,
3611                                 &wait, TASK_INTERRUPTIBLE);
3612                 for (i = 0; i < count; i++) {
3613                         obj_priv = to_intel_bo(object_list[i]);
3614                         if (atomic_read(&obj_priv->pending_flip) > 0)
3615                                 break;
3616                 }
3617                 if (i == count)
3618                         break;
3619
3620                 if (!signal_pending(current)) {
3621                         mutex_unlock(&dev->struct_mutex);
3622                         schedule();
3623                         mutex_lock(&dev->struct_mutex);
3624                         continue;
3625                 }
3626                 ret = -ERESTARTSYS;
3627                 break;
3628         }
3629         finish_wait(&dev_priv->pending_flip_queue, &wait);
3630
3631         return ret;
3632 }
3633
3634 static int
3635 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3636                        struct drm_file *file_priv,
3637                        struct drm_i915_gem_execbuffer2 *args,
3638                        struct drm_i915_gem_exec_object2 *exec_list)
3639 {
3640         drm_i915_private_t *dev_priv = dev->dev_private;
3641         struct drm_gem_object **object_list = NULL;
3642         struct drm_gem_object *batch_obj;
3643         struct drm_i915_gem_object *obj_priv;
3644         struct drm_clip_rect *cliprects = NULL;
3645         struct drm_i915_gem_relocation_entry *relocs = NULL;
3646         struct drm_i915_gem_request *request = NULL;
3647         int ret, ret2, i, pinned = 0;
3648         uint64_t exec_offset;
3649         uint32_t reloc_index;
3650         int pin_tries, flips;
3651
3652         struct intel_ring_buffer *ring = NULL;
3653
3654         ret = i915_gem_check_is_wedged(dev);
3655         if (ret)
3656                 return ret;
3657
3658 #if WATCH_EXEC
3659         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3660                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3661 #endif
3662         if (args->flags & I915_EXEC_BSD) {
3663                 if (!HAS_BSD(dev)) {
3664                         DRM_ERROR("execbuf with wrong flag\n");
3665                         return -EINVAL;
3666                 }
3667                 ring = &dev_priv->bsd_ring;
3668         } else {
3669                 ring = &dev_priv->render_ring;
3670         }
3671
3672         if (args->buffer_count < 1) {
3673                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3674                 return -EINVAL;
3675         }
3676         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3677         if (object_list == NULL) {
3678                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3679                           args->buffer_count);
3680                 ret = -ENOMEM;
3681                 goto pre_mutex_err;
3682         }
3683
3684         if (args->num_cliprects != 0) {
3685                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3686                                     GFP_KERNEL);
3687                 if (cliprects == NULL) {
3688                         ret = -ENOMEM;
3689                         goto pre_mutex_err;
3690                 }
3691
3692                 ret = copy_from_user(cliprects,
3693                                      (struct drm_clip_rect __user *)
3694                                      (uintptr_t) args->cliprects_ptr,
3695                                      sizeof(*cliprects) * args->num_cliprects);
3696                 if (ret != 0) {
3697                         DRM_ERROR("copy %d cliprects failed: %d\n",
3698                                   args->num_cliprects, ret);
3699                         ret = -EFAULT;
3700                         goto pre_mutex_err;
3701                 }
3702         }
3703
3704         request = kzalloc(sizeof(*request), GFP_KERNEL);
3705         if (request == NULL) {
3706                 ret = -ENOMEM;
3707                 goto pre_mutex_err;
3708         }
3709
3710         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3711                                             &relocs);
3712         if (ret != 0)
3713                 goto pre_mutex_err;
3714
3715         ret = i915_mutex_lock_interruptible(dev);
3716         if (ret)
3717                 goto pre_mutex_err;
3718
3719         i915_verify_inactive(dev, __FILE__, __LINE__);
3720
3721         if (dev_priv->mm.suspended) {
3722                 mutex_unlock(&dev->struct_mutex);
3723                 ret = -EBUSY;
3724                 goto pre_mutex_err;
3725         }
3726
3727         /* Look up object handles */
3728         flips = 0;
3729         for (i = 0; i < args->buffer_count; i++) {
3730                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3731                                                        exec_list[i].handle);
3732                 if (object_list[i] == NULL) {
3733                         DRM_ERROR("Invalid object handle %d at index %d\n",
3734                                    exec_list[i].handle, i);
3735                         /* prevent error path from reading uninitialized data */
3736                         args->buffer_count = i + 1;
3737                         ret = -ENOENT;
3738                         goto err;
3739                 }
3740
3741                 obj_priv = to_intel_bo(object_list[i]);
3742                 if (obj_priv->in_execbuffer) {
3743                         DRM_ERROR("Object %p appears more than once in object list\n",
3744                                    object_list[i]);
3745                         /* prevent error path from reading uninitialized data */
3746                         args->buffer_count = i + 1;
3747                         ret = -EINVAL;
3748                         goto err;
3749                 }
3750                 obj_priv->in_execbuffer = true;
3751                 flips += atomic_read(&obj_priv->pending_flip);
3752         }
3753
3754         if (flips > 0) {
3755                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3756                                                      args->buffer_count);
3757                 if (ret)
3758                         goto err;
3759         }
3760
3761         /* Pin and relocate */
3762         for (pin_tries = 0; ; pin_tries++) {
3763                 ret = 0;
3764                 reloc_index = 0;
3765
3766                 for (i = 0; i < args->buffer_count; i++) {
3767                         object_list[i]->pending_read_domains = 0;
3768                         object_list[i]->pending_write_domain = 0;
3769                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3770                                                                file_priv,
3771                                                                &exec_list[i],
3772                                                                &relocs[reloc_index]);
3773                         if (ret)
3774                                 break;
3775                         pinned = i + 1;
3776                         reloc_index += exec_list[i].relocation_count;
3777                 }
3778                 /* success */
3779                 if (ret == 0)
3780                         break;
3781
3782                 /* error other than GTT full, or we've already tried again */
3783                 if (ret != -ENOSPC || pin_tries >= 1) {
3784                         if (ret != -ERESTARTSYS) {
3785                                 unsigned long long total_size = 0;
3786                                 int num_fences = 0;
3787                                 for (i = 0; i < args->buffer_count; i++) {
3788                                         obj_priv = to_intel_bo(object_list[i]);
3789
3790                                         total_size += object_list[i]->size;
3791                                         num_fences +=
3792                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3793                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3794                                 }
3795                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3796                                           pinned+1, args->buffer_count,
3797                                           total_size, num_fences,
3798                                           ret);
3799                                 DRM_ERROR("%d objects [%d pinned], "
3800                                           "%d object bytes [%d pinned], "
3801                                           "%d/%d gtt bytes\n",
3802                                           atomic_read(&dev->object_count),
3803                                           atomic_read(&dev->pin_count),
3804                                           atomic_read(&dev->object_memory),
3805                                           atomic_read(&dev->pin_memory),
3806                                           atomic_read(&dev->gtt_memory),
3807                                           dev->gtt_total);
3808                         }
3809                         goto err;
3810                 }
3811
3812                 /* unpin all of our buffers */
3813                 for (i = 0; i < pinned; i++)
3814                         i915_gem_object_unpin(object_list[i]);
3815                 pinned = 0;
3816
3817                 /* evict everyone we can from the aperture */
3818                 ret = i915_gem_evict_everything(dev);
3819                 if (ret && ret != -ENOSPC)
3820                         goto err;
3821         }
3822
3823         /* Set the pending read domains for the batch buffer to COMMAND */
3824         batch_obj = object_list[args->buffer_count-1];
3825         if (batch_obj->pending_write_domain) {
3826                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3827                 ret = -EINVAL;
3828                 goto err;
3829         }
3830         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3831
3832         /* Sanity check the batch buffer, prior to moving objects */
3833         exec_offset = exec_list[args->buffer_count - 1].offset;
3834         ret = i915_gem_check_execbuffer (args, exec_offset);
3835         if (ret != 0) {
3836                 DRM_ERROR("execbuf with invalid offset/length\n");
3837                 goto err;
3838         }
3839
3840         i915_verify_inactive(dev, __FILE__, __LINE__);
3841
3842         /* Zero the global flush/invalidate flags. These
3843          * will be modified as new domains are computed
3844          * for each object
3845          */
3846         dev->invalidate_domains = 0;
3847         dev->flush_domains = 0;
3848         dev_priv->mm.flush_rings = 0;
3849
3850         for (i = 0; i < args->buffer_count; i++) {
3851                 struct drm_gem_object *obj = object_list[i];
3852
3853                 /* Compute new gpu domains and update invalidate/flush */
3854                 i915_gem_object_set_to_gpu_domain(obj);
3855         }
3856
3857         i915_verify_inactive(dev, __FILE__, __LINE__);
3858
3859         if (dev->invalidate_domains | dev->flush_domains) {
3860 #if WATCH_EXEC
3861                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3862                           __func__,
3863                          dev->invalidate_domains,
3864                          dev->flush_domains);
3865 #endif
3866                 i915_gem_flush(dev, file_priv,
3867                                dev->invalidate_domains,
3868                                dev->flush_domains,
3869                                dev_priv->mm.flush_rings);
3870         }
3871
3872         for (i = 0; i < args->buffer_count; i++) {
3873                 struct drm_gem_object *obj = object_list[i];
3874                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3875                 uint32_t old_write_domain = obj->write_domain;
3876
3877                 obj->write_domain = obj->pending_write_domain;
3878                 if (obj->write_domain)
3879                         list_move_tail(&obj_priv->gpu_write_list,
3880                                        &dev_priv->mm.gpu_write_list);
3881                 else
3882                         list_del_init(&obj_priv->gpu_write_list);
3883
3884                 trace_i915_gem_object_change_domain(obj,
3885                                                     obj->read_domains,
3886                                                     old_write_domain);
3887         }
3888
3889         i915_verify_inactive(dev, __FILE__, __LINE__);
3890
3891 #if WATCH_COHERENCY
3892         for (i = 0; i < args->buffer_count; i++) {
3893                 i915_gem_object_check_coherency(object_list[i],
3894                                                 exec_list[i].handle);
3895         }
3896 #endif
3897
3898 #if WATCH_EXEC
3899         i915_gem_dump_object(batch_obj,
3900                               args->batch_len,
3901                               __func__,
3902                               ~0);
3903 #endif
3904
3905         /* Exec the batchbuffer */
3906         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3907                         cliprects, exec_offset);
3908         if (ret) {
3909                 DRM_ERROR("dispatch failed %d\n", ret);
3910                 goto err;
3911         }
3912
3913         /*
3914          * Ensure that the commands in the batch buffer are
3915          * finished before the interrupt fires
3916          */
3917         i915_retire_commands(dev, ring);
3918
3919         i915_verify_inactive(dev, __FILE__, __LINE__);
3920
3921         for (i = 0; i < args->buffer_count; i++) {
3922                 struct drm_gem_object *obj = object_list[i];
3923                 obj_priv = to_intel_bo(obj);
3924
3925                 i915_gem_object_move_to_active(obj, ring);
3926 #if WATCH_LRU
3927                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3928 #endif
3929         }
3930         i915_add_request(dev, file_priv, request, ring);
3931         request = NULL;
3932
3933 #if WATCH_LRU
3934         i915_dump_lru(dev, __func__);
3935 #endif
3936
3937         i915_verify_inactive(dev, __FILE__, __LINE__);
3938
3939 err:
3940         for (i = 0; i < pinned; i++)
3941                 i915_gem_object_unpin(object_list[i]);
3942
3943         for (i = 0; i < args->buffer_count; i++) {
3944                 if (object_list[i]) {
3945                         obj_priv = to_intel_bo(object_list[i]);
3946                         obj_priv->in_execbuffer = false;
3947                 }
3948                 drm_gem_object_unreference(object_list[i]);
3949         }
3950
3951         mutex_unlock(&dev->struct_mutex);
3952
3953 pre_mutex_err:
3954         /* Copy the updated relocations out regardless of current error
3955          * state.  Failure to update the relocs would mean that the next
3956          * time userland calls execbuf, it would do so with presumed offset
3957          * state that didn't match the actual object state.
3958          */
3959         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3960                                            relocs);
3961         if (ret2 != 0) {
3962                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3963
3964                 if (ret == 0)
3965                         ret = ret2;
3966         }
3967
3968         drm_free_large(object_list);
3969         kfree(cliprects);
3970         kfree(request);
3971
3972         return ret;
3973 }
3974
3975 /*
3976  * Legacy execbuffer just creates an exec2 list from the original exec object
3977  * list array and passes it to the real function.
3978  */
3979 int
3980 i915_gem_execbuffer(struct drm_device *dev, void *data,
3981                     struct drm_file *file_priv)
3982 {
3983         struct drm_i915_gem_execbuffer *args = data;
3984         struct drm_i915_gem_execbuffer2 exec2;
3985         struct drm_i915_gem_exec_object *exec_list = NULL;
3986         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3987         int ret, i;
3988
3989 #if WATCH_EXEC
3990         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3991                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3992 #endif
3993
3994         if (args->buffer_count < 1) {
3995                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3996                 return -EINVAL;
3997         }
3998
3999         /* Copy in the exec list from userland */
4000         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4001         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4002         if (exec_list == NULL || exec2_list == NULL) {
4003                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4004                           args->buffer_count);
4005                 drm_free_large(exec_list);
4006                 drm_free_large(exec2_list);
4007                 return -ENOMEM;
4008         }
4009         ret = copy_from_user(exec_list,
4010                              (struct drm_i915_relocation_entry __user *)
4011                              (uintptr_t) args->buffers_ptr,
4012                              sizeof(*exec_list) * args->buffer_count);
4013         if (ret != 0) {
4014                 DRM_ERROR("copy %d exec entries failed %d\n",
4015                           args->buffer_count, ret);
4016                 drm_free_large(exec_list);
4017                 drm_free_large(exec2_list);
4018                 return -EFAULT;
4019         }
4020
4021         for (i = 0; i < args->buffer_count; i++) {
4022                 exec2_list[i].handle = exec_list[i].handle;
4023                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4024                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4025                 exec2_list[i].alignment = exec_list[i].alignment;
4026                 exec2_list[i].offset = exec_list[i].offset;
4027                 if (INTEL_INFO(dev)->gen < 4)
4028                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4029                 else
4030                         exec2_list[i].flags = 0;
4031         }
4032
4033         exec2.buffers_ptr = args->buffers_ptr;
4034         exec2.buffer_count = args->buffer_count;
4035         exec2.batch_start_offset = args->batch_start_offset;
4036         exec2.batch_len = args->batch_len;
4037         exec2.DR1 = args->DR1;
4038         exec2.DR4 = args->DR4;
4039         exec2.num_cliprects = args->num_cliprects;
4040         exec2.cliprects_ptr = args->cliprects_ptr;
4041         exec2.flags = I915_EXEC_RENDER;
4042
4043         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4044         if (!ret) {
4045                 /* Copy the new buffer offsets back to the user's exec list. */
4046                 for (i = 0; i < args->buffer_count; i++)
4047                         exec_list[i].offset = exec2_list[i].offset;
4048                 /* ... and back out to userspace */
4049                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4050                                    (uintptr_t) args->buffers_ptr,
4051                                    exec_list,
4052                                    sizeof(*exec_list) * args->buffer_count);
4053                 if (ret) {
4054                         ret = -EFAULT;
4055                         DRM_ERROR("failed to copy %d exec entries "
4056                                   "back to user (%d)\n",
4057                                   args->buffer_count, ret);
4058                 }
4059         }
4060
4061         drm_free_large(exec_list);
4062         drm_free_large(exec2_list);
4063         return ret;
4064 }
4065
4066 int
4067 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4068                      struct drm_file *file_priv)
4069 {
4070         struct drm_i915_gem_execbuffer2 *args = data;
4071         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4072         int ret;
4073
4074 #if WATCH_EXEC
4075         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4076                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4077 #endif
4078
4079         if (args->buffer_count < 1) {
4080                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4081                 return -EINVAL;
4082         }
4083
4084         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4085         if (exec2_list == NULL) {
4086                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4087                           args->buffer_count);
4088                 return -ENOMEM;
4089         }
4090         ret = copy_from_user(exec2_list,
4091                              (struct drm_i915_relocation_entry __user *)
4092                              (uintptr_t) args->buffers_ptr,
4093                              sizeof(*exec2_list) * args->buffer_count);
4094         if (ret != 0) {
4095                 DRM_ERROR("copy %d exec entries failed %d\n",
4096                           args->buffer_count, ret);
4097                 drm_free_large(exec2_list);
4098                 return -EFAULT;
4099         }
4100
4101         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4102         if (!ret) {
4103                 /* Copy the new buffer offsets back to the user's exec list. */
4104                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4105                                    (uintptr_t) args->buffers_ptr,
4106                                    exec2_list,
4107                                    sizeof(*exec2_list) * args->buffer_count);
4108                 if (ret) {
4109                         ret = -EFAULT;
4110                         DRM_ERROR("failed to copy %d exec entries "
4111                                   "back to user (%d)\n",
4112                                   args->buffer_count, ret);
4113                 }
4114         }
4115
4116         drm_free_large(exec2_list);
4117         return ret;
4118 }
4119
4120 int
4121 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4122 {
4123         struct drm_device *dev = obj->dev;
4124         struct drm_i915_private *dev_priv = dev->dev_private;
4125         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4126         int ret;
4127
4128         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4129
4130         i915_verify_inactive(dev, __FILE__, __LINE__);
4131
4132         if (obj_priv->gtt_space != NULL) {
4133                 if (alignment == 0)
4134                         alignment = i915_gem_get_gtt_alignment(obj);
4135                 if (obj_priv->gtt_offset & (alignment - 1)) {
4136                         WARN(obj_priv->pin_count,
4137                              "bo is already pinned with incorrect alignment:"
4138                              " offset=%x, req.alignment=%x\n",
4139                              obj_priv->gtt_offset, alignment);
4140                         ret = i915_gem_object_unbind(obj);
4141                         if (ret)
4142                                 return ret;
4143                 }
4144         }
4145
4146         if (obj_priv->gtt_space == NULL) {
4147                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4148                 if (ret)
4149                         return ret;
4150         }
4151
4152         obj_priv->pin_count++;
4153
4154         /* If the object is not active and not pending a flush,
4155          * remove it from the inactive list
4156          */
4157         if (obj_priv->pin_count == 1) {
4158                 atomic_inc(&dev->pin_count);
4159                 atomic_add(obj->size, &dev->pin_memory);
4160                 if (!obj_priv->active)
4161                         list_move_tail(&obj_priv->list,
4162                                        &dev_priv->mm.pinned_list);
4163         }
4164         i915_verify_inactive(dev, __FILE__, __LINE__);
4165
4166         return 0;
4167 }
4168
4169 void
4170 i915_gem_object_unpin(struct drm_gem_object *obj)
4171 {
4172         struct drm_device *dev = obj->dev;
4173         drm_i915_private_t *dev_priv = dev->dev_private;
4174         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4175
4176         i915_verify_inactive(dev, __FILE__, __LINE__);
4177         obj_priv->pin_count--;
4178         BUG_ON(obj_priv->pin_count < 0);
4179         BUG_ON(obj_priv->gtt_space == NULL);
4180
4181         /* If the object is no longer pinned, and is
4182          * neither active nor being flushed, then stick it on
4183          * the inactive list
4184          */
4185         if (obj_priv->pin_count == 0) {
4186                 if (!obj_priv->active)
4187                         list_move_tail(&obj_priv->list,
4188                                        &dev_priv->mm.inactive_list);
4189                 atomic_dec(&dev->pin_count);
4190                 atomic_sub(obj->size, &dev->pin_memory);
4191         }
4192         i915_verify_inactive(dev, __FILE__, __LINE__);
4193 }
4194
4195 int
4196 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4197                    struct drm_file *file_priv)
4198 {
4199         struct drm_i915_gem_pin *args = data;
4200         struct drm_gem_object *obj;
4201         struct drm_i915_gem_object *obj_priv;
4202         int ret;
4203
4204         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4205         if (obj == NULL) {
4206                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4207                           args->handle);
4208                 return -ENOENT;
4209         }
4210         obj_priv = to_intel_bo(obj);
4211
4212         ret = i915_mutex_lock_interruptible(dev);
4213         if (ret) {
4214                 drm_gem_object_unreference_unlocked(obj);
4215                 return ret;
4216         }
4217
4218         if (obj_priv->madv != I915_MADV_WILLNEED) {
4219                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4220                 drm_gem_object_unreference(obj);
4221                 mutex_unlock(&dev->struct_mutex);
4222                 return -EINVAL;
4223         }
4224
4225         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4226                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4227                           args->handle);
4228                 drm_gem_object_unreference(obj);
4229                 mutex_unlock(&dev->struct_mutex);
4230                 return -EINVAL;
4231         }
4232
4233         obj_priv->user_pin_count++;
4234         obj_priv->pin_filp = file_priv;
4235         if (obj_priv->user_pin_count == 1) {
4236                 ret = i915_gem_object_pin(obj, args->alignment);
4237                 if (ret != 0) {
4238                         drm_gem_object_unreference(obj);
4239                         mutex_unlock(&dev->struct_mutex);
4240                         return ret;
4241                 }
4242         }
4243
4244         /* XXX - flush the CPU caches for pinned objects
4245          * as the X server doesn't manage domains yet
4246          */
4247         i915_gem_object_flush_cpu_write_domain(obj);
4248         args->offset = obj_priv->gtt_offset;
4249         drm_gem_object_unreference(obj);
4250         mutex_unlock(&dev->struct_mutex);
4251
4252         return 0;
4253 }
4254
4255 int
4256 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4257                      struct drm_file *file_priv)
4258 {
4259         struct drm_i915_gem_pin *args = data;
4260         struct drm_gem_object *obj;
4261         struct drm_i915_gem_object *obj_priv;
4262         int ret;
4263
4264         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4265         if (obj == NULL) {
4266                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4267                           args->handle);
4268                 return -ENOENT;
4269         }
4270
4271         obj_priv = to_intel_bo(obj);
4272
4273         ret = i915_mutex_lock_interruptible(dev);
4274         if (ret) {
4275                 drm_gem_object_unreference_unlocked(obj);
4276                 return ret;
4277         }
4278
4279         if (obj_priv->pin_filp != file_priv) {
4280                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4281                           args->handle);
4282                 drm_gem_object_unreference(obj);
4283                 mutex_unlock(&dev->struct_mutex);
4284                 return -EINVAL;
4285         }
4286         obj_priv->user_pin_count--;
4287         if (obj_priv->user_pin_count == 0) {
4288                 obj_priv->pin_filp = NULL;
4289                 i915_gem_object_unpin(obj);
4290         }
4291
4292         drm_gem_object_unreference(obj);
4293         mutex_unlock(&dev->struct_mutex);
4294         return 0;
4295 }
4296
4297 int
4298 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4299                     struct drm_file *file_priv)
4300 {
4301         struct drm_i915_gem_busy *args = data;
4302         struct drm_gem_object *obj;
4303         struct drm_i915_gem_object *obj_priv;
4304         int ret;
4305
4306         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4307         if (obj == NULL) {
4308                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4309                           args->handle);
4310                 return -ENOENT;
4311         }
4312
4313         ret = i915_mutex_lock_interruptible(dev);
4314         if (ret) {
4315                 drm_gem_object_unreference_unlocked(obj);
4316                 return ret;
4317         }
4318
4319         /* Count all active objects as busy, even if they are currently not used
4320          * by the gpu. Users of this interface expect objects to eventually
4321          * become non-busy without any further actions, therefore emit any
4322          * necessary flushes here.
4323          */
4324         obj_priv = to_intel_bo(obj);
4325         args->busy = obj_priv->active;
4326         if (args->busy) {
4327                 /* Unconditionally flush objects, even when the gpu still uses this
4328                  * object. Userspace calling this function indicates that it wants to
4329                  * use this buffer rather sooner than later, so issuing the required
4330                  * flush earlier is beneficial.
4331                  */
4332                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4333                         i915_gem_flush_ring(dev, file_priv,
4334                                             obj_priv->ring,
4335                                             0, obj->write_domain);
4336
4337                 /* Update the active list for the hardware's current position.
4338                  * Otherwise this only updates on a delayed timer or when irqs
4339                  * are actually unmasked, and our working set ends up being
4340                  * larger than required.
4341                  */
4342                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4343
4344                 args->busy = obj_priv->active;
4345         }
4346
4347         drm_gem_object_unreference(obj);
4348         mutex_unlock(&dev->struct_mutex);
4349         return 0;
4350 }
4351
4352 int
4353 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4354                         struct drm_file *file_priv)
4355 {
4356     return i915_gem_ring_throttle(dev, file_priv);
4357 }
4358
4359 int
4360 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4361                        struct drm_file *file_priv)
4362 {
4363         struct drm_i915_gem_madvise *args = data;
4364         struct drm_gem_object *obj;
4365         struct drm_i915_gem_object *obj_priv;
4366         int ret;
4367
4368         switch (args->madv) {
4369         case I915_MADV_DONTNEED:
4370         case I915_MADV_WILLNEED:
4371             break;
4372         default:
4373             return -EINVAL;
4374         }
4375
4376         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4377         if (obj == NULL) {
4378                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4379                           args->handle);
4380                 return -ENOENT;
4381         }
4382         obj_priv = to_intel_bo(obj);
4383
4384         ret = i915_mutex_lock_interruptible(dev);
4385         if (ret) {
4386                 drm_gem_object_unreference_unlocked(obj);
4387                 return ret;
4388         }
4389
4390         if (obj_priv->pin_count) {
4391                 drm_gem_object_unreference(obj);
4392                 mutex_unlock(&dev->struct_mutex);
4393
4394                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4395                 return -EINVAL;
4396         }
4397
4398         if (obj_priv->madv != __I915_MADV_PURGED)
4399                 obj_priv->madv = args->madv;
4400
4401         /* if the object is no longer bound, discard its backing storage */
4402         if (i915_gem_object_is_purgeable(obj_priv) &&
4403             obj_priv->gtt_space == NULL)
4404                 i915_gem_object_truncate(obj);
4405
4406         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4407
4408         drm_gem_object_unreference(obj);
4409         mutex_unlock(&dev->struct_mutex);
4410
4411         return 0;
4412 }
4413
4414 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4415                                               size_t size)
4416 {
4417         struct drm_i915_gem_object *obj;
4418
4419         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4420         if (obj == NULL)
4421                 return NULL;
4422
4423         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4424                 kfree(obj);
4425                 return NULL;
4426         }
4427
4428         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4429         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4430
4431         obj->agp_type = AGP_USER_MEMORY;
4432         obj->base.driver_private = NULL;
4433         obj->fence_reg = I915_FENCE_REG_NONE;
4434         INIT_LIST_HEAD(&obj->list);
4435         INIT_LIST_HEAD(&obj->gpu_write_list);
4436         obj->madv = I915_MADV_WILLNEED;
4437
4438         trace_i915_gem_object_create(&obj->base);
4439
4440         return &obj->base;
4441 }
4442
4443 int i915_gem_init_object(struct drm_gem_object *obj)
4444 {
4445         BUG();
4446
4447         return 0;
4448 }
4449
4450 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4451 {
4452         struct drm_device *dev = obj->dev;
4453         drm_i915_private_t *dev_priv = dev->dev_private;
4454         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4455         int ret;
4456
4457         ret = i915_gem_object_unbind(obj);
4458         if (ret == -ERESTARTSYS) {
4459                 list_move(&obj_priv->list,
4460                           &dev_priv->mm.deferred_free_list);
4461                 return;
4462         }
4463
4464         if (obj_priv->mmap_offset)
4465                 i915_gem_free_mmap_offset(obj);
4466
4467         drm_gem_object_release(obj);
4468
4469         kfree(obj_priv->page_cpu_valid);
4470         kfree(obj_priv->bit_17);
4471         kfree(obj_priv);
4472 }
4473
4474 void i915_gem_free_object(struct drm_gem_object *obj)
4475 {
4476         struct drm_device *dev = obj->dev;
4477         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4478
4479         trace_i915_gem_object_destroy(obj);
4480
4481         while (obj_priv->pin_count > 0)
4482                 i915_gem_object_unpin(obj);
4483
4484         if (obj_priv->phys_obj)
4485                 i915_gem_detach_phys_object(dev, obj);
4486
4487         i915_gem_free_object_tail(obj);
4488 }
4489
4490 int
4491 i915_gem_idle(struct drm_device *dev)
4492 {
4493         drm_i915_private_t *dev_priv = dev->dev_private;
4494         int ret;
4495
4496         mutex_lock(&dev->struct_mutex);
4497
4498         if (dev_priv->mm.suspended ||
4499                         (dev_priv->render_ring.gem_object == NULL) ||
4500                         (HAS_BSD(dev) &&
4501                          dev_priv->bsd_ring.gem_object == NULL)) {
4502                 mutex_unlock(&dev->struct_mutex);
4503                 return 0;
4504         }
4505
4506         ret = i915_gpu_idle(dev);
4507         if (ret) {
4508                 mutex_unlock(&dev->struct_mutex);
4509                 return ret;
4510         }
4511
4512         /* Under UMS, be paranoid and evict. */
4513         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4514                 ret = i915_gem_evict_inactive(dev);
4515                 if (ret) {
4516                         mutex_unlock(&dev->struct_mutex);
4517                         return ret;
4518                 }
4519         }
4520
4521         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4522          * We need to replace this with a semaphore, or something.
4523          * And not confound mm.suspended!
4524          */
4525         dev_priv->mm.suspended = 1;
4526         del_timer_sync(&dev_priv->hangcheck_timer);
4527
4528         i915_kernel_lost_context(dev);
4529         i915_gem_cleanup_ringbuffer(dev);
4530
4531         mutex_unlock(&dev->struct_mutex);
4532
4533         /* Cancel the retire work handler, which should be idle now. */
4534         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4535
4536         return 0;
4537 }
4538
4539 /*
4540  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4541  * over cache flushing.
4542  */
4543 static int
4544 i915_gem_init_pipe_control(struct drm_device *dev)
4545 {
4546         drm_i915_private_t *dev_priv = dev->dev_private;
4547         struct drm_gem_object *obj;
4548         struct drm_i915_gem_object *obj_priv;
4549         int ret;
4550
4551         obj = i915_gem_alloc_object(dev, 4096);
4552         if (obj == NULL) {
4553                 DRM_ERROR("Failed to allocate seqno page\n");
4554                 ret = -ENOMEM;
4555                 goto err;
4556         }
4557         obj_priv = to_intel_bo(obj);
4558         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4559
4560         ret = i915_gem_object_pin(obj, 4096);
4561         if (ret)
4562                 goto err_unref;
4563
4564         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4565         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4566         if (dev_priv->seqno_page == NULL)
4567                 goto err_unpin;
4568
4569         dev_priv->seqno_obj = obj;
4570         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4571
4572         return 0;
4573
4574 err_unpin:
4575         i915_gem_object_unpin(obj);
4576 err_unref:
4577         drm_gem_object_unreference(obj);
4578 err:
4579         return ret;
4580 }
4581
4582
4583 static void
4584 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4585 {
4586         drm_i915_private_t *dev_priv = dev->dev_private;
4587         struct drm_gem_object *obj;
4588         struct drm_i915_gem_object *obj_priv;
4589
4590         obj = dev_priv->seqno_obj;
4591         obj_priv = to_intel_bo(obj);
4592         kunmap(obj_priv->pages[0]);
4593         i915_gem_object_unpin(obj);
4594         drm_gem_object_unreference(obj);
4595         dev_priv->seqno_obj = NULL;
4596
4597         dev_priv->seqno_page = NULL;
4598 }
4599
4600 int
4601 i915_gem_init_ringbuffer(struct drm_device *dev)
4602 {
4603         drm_i915_private_t *dev_priv = dev->dev_private;
4604         int ret;
4605
4606         if (HAS_PIPE_CONTROL(dev)) {
4607                 ret = i915_gem_init_pipe_control(dev);
4608                 if (ret)
4609                         return ret;
4610         }
4611
4612         ret = intel_init_render_ring_buffer(dev);
4613         if (ret)
4614                 goto cleanup_pipe_control;
4615
4616         if (HAS_BSD(dev)) {
4617                 ret = intel_init_bsd_ring_buffer(dev);
4618                 if (ret)
4619                         goto cleanup_render_ring;
4620         }
4621
4622         dev_priv->next_seqno = 1;
4623
4624         return 0;
4625
4626 cleanup_render_ring:
4627         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4628 cleanup_pipe_control:
4629         if (HAS_PIPE_CONTROL(dev))
4630                 i915_gem_cleanup_pipe_control(dev);
4631         return ret;
4632 }
4633
4634 void
4635 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4636 {
4637         drm_i915_private_t *dev_priv = dev->dev_private;
4638
4639         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4640         if (HAS_BSD(dev))
4641                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4642         if (HAS_PIPE_CONTROL(dev))
4643                 i915_gem_cleanup_pipe_control(dev);
4644 }
4645
4646 int
4647 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4648                        struct drm_file *file_priv)
4649 {
4650         drm_i915_private_t *dev_priv = dev->dev_private;
4651         int ret;
4652
4653         if (drm_core_check_feature(dev, DRIVER_MODESET))
4654                 return 0;
4655
4656         if (atomic_read(&dev_priv->mm.wedged)) {
4657                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4658                 atomic_set(&dev_priv->mm.wedged, 0);
4659         }
4660
4661         mutex_lock(&dev->struct_mutex);
4662         dev_priv->mm.suspended = 0;
4663
4664         ret = i915_gem_init_ringbuffer(dev);
4665         if (ret != 0) {
4666                 mutex_unlock(&dev->struct_mutex);
4667                 return ret;
4668         }
4669
4670         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4671         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4672         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4673         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4674         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4675         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4676         mutex_unlock(&dev->struct_mutex);
4677
4678         ret = drm_irq_install(dev);
4679         if (ret)
4680                 goto cleanup_ringbuffer;
4681
4682         return 0;
4683
4684 cleanup_ringbuffer:
4685         mutex_lock(&dev->struct_mutex);
4686         i915_gem_cleanup_ringbuffer(dev);
4687         dev_priv->mm.suspended = 1;
4688         mutex_unlock(&dev->struct_mutex);
4689
4690         return ret;
4691 }
4692
4693 int
4694 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4695                        struct drm_file *file_priv)
4696 {
4697         if (drm_core_check_feature(dev, DRIVER_MODESET))
4698                 return 0;
4699
4700         drm_irq_uninstall(dev);
4701         return i915_gem_idle(dev);
4702 }
4703
4704 void
4705 i915_gem_lastclose(struct drm_device *dev)
4706 {
4707         int ret;
4708
4709         if (drm_core_check_feature(dev, DRIVER_MODESET))
4710                 return;
4711
4712         ret = i915_gem_idle(dev);
4713         if (ret)
4714                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4715 }
4716
4717 void
4718 i915_gem_load(struct drm_device *dev)
4719 {
4720         int i;
4721         drm_i915_private_t *dev_priv = dev->dev_private;
4722
4723         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4724         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4725         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4726         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4727         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4728         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4729         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4730         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4731         if (HAS_BSD(dev)) {
4732                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4733                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4734         }
4735         for (i = 0; i < 16; i++)
4736                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4737         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4738                           i915_gem_retire_work_handler);
4739         init_completion(&dev_priv->error_completion);
4740         spin_lock(&shrink_list_lock);
4741         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4742         spin_unlock(&shrink_list_lock);
4743
4744         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4745         if (IS_GEN3(dev)) {
4746                 u32 tmp = I915_READ(MI_ARB_STATE);
4747                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4748                         /* arb state is a masked write, so set bit + bit in mask */
4749                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4750                         I915_WRITE(MI_ARB_STATE, tmp);
4751                 }
4752         }
4753
4754         /* Old X drivers will take 0-2 for front, back, depth buffers */
4755         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4756                 dev_priv->fence_reg_start = 3;
4757
4758         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4759                 dev_priv->num_fence_regs = 16;
4760         else
4761                 dev_priv->num_fence_regs = 8;
4762
4763         /* Initialize fence registers to zero */
4764         switch (INTEL_INFO(dev)->gen) {
4765         case 6:
4766                 for (i = 0; i < 16; i++)
4767                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4768                 break;
4769         case 5:
4770         case 4:
4771                 for (i = 0; i < 16; i++)
4772                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4773                 break;
4774         case 3:
4775                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4776                         for (i = 0; i < 8; i++)
4777                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4778         case 2:
4779                 for (i = 0; i < 8; i++)
4780                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4781                 break;
4782         }
4783         i915_gem_detect_bit_6_swizzle(dev);
4784         init_waitqueue_head(&dev_priv->pending_flip_queue);
4785 }
4786
4787 /*
4788  * Create a physically contiguous memory object for this object
4789  * e.g. for cursor + overlay regs
4790  */
4791 static int i915_gem_init_phys_object(struct drm_device *dev,
4792                                      int id, int size, int align)
4793 {
4794         drm_i915_private_t *dev_priv = dev->dev_private;
4795         struct drm_i915_gem_phys_object *phys_obj;
4796         int ret;
4797
4798         if (dev_priv->mm.phys_objs[id - 1] || !size)
4799                 return 0;
4800
4801         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4802         if (!phys_obj)
4803                 return -ENOMEM;
4804
4805         phys_obj->id = id;
4806
4807         phys_obj->handle = drm_pci_alloc(dev, size, align);
4808         if (!phys_obj->handle) {
4809                 ret = -ENOMEM;
4810                 goto kfree_obj;
4811         }
4812 #ifdef CONFIG_X86
4813         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4814 #endif
4815
4816         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4817
4818         return 0;
4819 kfree_obj:
4820         kfree(phys_obj);
4821         return ret;
4822 }
4823
4824 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4825 {
4826         drm_i915_private_t *dev_priv = dev->dev_private;
4827         struct drm_i915_gem_phys_object *phys_obj;
4828
4829         if (!dev_priv->mm.phys_objs[id - 1])
4830                 return;
4831
4832         phys_obj = dev_priv->mm.phys_objs[id - 1];
4833         if (phys_obj->cur_obj) {
4834                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4835         }
4836
4837 #ifdef CONFIG_X86
4838         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4839 #endif
4840         drm_pci_free(dev, phys_obj->handle);
4841         kfree(phys_obj);
4842         dev_priv->mm.phys_objs[id - 1] = NULL;
4843 }
4844
4845 void i915_gem_free_all_phys_object(struct drm_device *dev)
4846 {
4847         int i;
4848
4849         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4850                 i915_gem_free_phys_object(dev, i);
4851 }
4852
4853 void i915_gem_detach_phys_object(struct drm_device *dev,
4854                                  struct drm_gem_object *obj)
4855 {
4856         struct drm_i915_gem_object *obj_priv;
4857         int i;
4858         int ret;
4859         int page_count;
4860
4861         obj_priv = to_intel_bo(obj);
4862         if (!obj_priv->phys_obj)
4863                 return;
4864
4865         ret = i915_gem_object_get_pages(obj, 0);
4866         if (ret)
4867                 goto out;
4868
4869         page_count = obj->size / PAGE_SIZE;
4870
4871         for (i = 0; i < page_count; i++) {
4872                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4873                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4874
4875                 memcpy(dst, src, PAGE_SIZE);
4876                 kunmap_atomic(dst, KM_USER0);
4877         }
4878         drm_clflush_pages(obj_priv->pages, page_count);
4879         drm_agp_chipset_flush(dev);
4880
4881         i915_gem_object_put_pages(obj);
4882 out:
4883         obj_priv->phys_obj->cur_obj = NULL;
4884         obj_priv->phys_obj = NULL;
4885 }
4886
4887 int
4888 i915_gem_attach_phys_object(struct drm_device *dev,
4889                             struct drm_gem_object *obj,
4890                             int id,
4891                             int align)
4892 {
4893         drm_i915_private_t *dev_priv = dev->dev_private;
4894         struct drm_i915_gem_object *obj_priv;
4895         int ret = 0;
4896         int page_count;
4897         int i;
4898
4899         if (id > I915_MAX_PHYS_OBJECT)
4900                 return -EINVAL;
4901
4902         obj_priv = to_intel_bo(obj);
4903
4904         if (obj_priv->phys_obj) {
4905                 if (obj_priv->phys_obj->id == id)
4906                         return 0;
4907                 i915_gem_detach_phys_object(dev, obj);
4908         }
4909
4910         /* create a new object */
4911         if (!dev_priv->mm.phys_objs[id - 1]) {
4912                 ret = i915_gem_init_phys_object(dev, id,
4913                                                 obj->size, align);
4914                 if (ret) {
4915                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4916                         goto out;
4917                 }
4918         }
4919
4920         /* bind to the object */
4921         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4922         obj_priv->phys_obj->cur_obj = obj;
4923
4924         ret = i915_gem_object_get_pages(obj, 0);
4925         if (ret) {
4926                 DRM_ERROR("failed to get page list\n");
4927                 goto out;
4928         }
4929
4930         page_count = obj->size / PAGE_SIZE;
4931
4932         for (i = 0; i < page_count; i++) {
4933                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4934                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4935
4936                 memcpy(dst, src, PAGE_SIZE);
4937                 kunmap_atomic(src, KM_USER0);
4938         }
4939
4940         i915_gem_object_put_pages(obj);
4941
4942         return 0;
4943 out:
4944         return ret;
4945 }
4946
4947 static int
4948 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4949                      struct drm_i915_gem_pwrite *args,
4950                      struct drm_file *file_priv)
4951 {
4952         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4953         void *obj_addr;
4954         int ret;
4955         char __user *user_data;
4956
4957         user_data = (char __user *) (uintptr_t) args->data_ptr;
4958         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4959
4960         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4961         ret = copy_from_user(obj_addr, user_data, args->size);
4962         if (ret)
4963                 return -EFAULT;
4964
4965         drm_agp_chipset_flush(dev);
4966         return 0;
4967 }
4968
4969 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4970 {
4971         struct drm_i915_file_private *file_priv = file->driver_priv;
4972
4973         /* Clean up our request list when the client is going away, so that
4974          * later retire_requests won't dereference our soon-to-be-gone
4975          * file_priv.
4976          */
4977         mutex_lock(&dev->struct_mutex);
4978         mutex_lock(&file_priv->mutex);
4979         while (!list_empty(&file_priv->mm.request_list)) {
4980                 struct drm_i915_gem_request *request;
4981
4982                 request = list_first_entry(&file_priv->mm.request_list,
4983                                            struct drm_i915_gem_request,
4984                                            client_list);
4985                 list_del(&request->client_list);
4986                 request->file_priv = NULL;
4987         }
4988         mutex_unlock(&file_priv->mutex);
4989         mutex_unlock(&dev->struct_mutex);
4990 }
4991
4992 static int
4993 i915_gpu_is_active(struct drm_device *dev)
4994 {
4995         drm_i915_private_t *dev_priv = dev->dev_private;
4996         int lists_empty;
4997
4998         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4999                       list_empty(&dev_priv->render_ring.active_list);
5000         if (HAS_BSD(dev))
5001                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
5002
5003         return !lists_empty;
5004 }
5005
5006 static int
5007 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
5008 {
5009         drm_i915_private_t *dev_priv, *next_dev;
5010         struct drm_i915_gem_object *obj_priv, *next_obj;
5011         int cnt = 0;
5012         int would_deadlock = 1;
5013
5014         /* "fast-path" to count number of available objects */
5015         if (nr_to_scan == 0) {
5016                 spin_lock(&shrink_list_lock);
5017                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5018                         struct drm_device *dev = dev_priv->dev;
5019
5020                         if (mutex_trylock(&dev->struct_mutex)) {
5021                                 list_for_each_entry(obj_priv,
5022                                                     &dev_priv->mm.inactive_list,
5023                                                     list)
5024                                         cnt++;
5025                                 mutex_unlock(&dev->struct_mutex);
5026                         }
5027                 }
5028                 spin_unlock(&shrink_list_lock);
5029
5030                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5031         }
5032
5033         spin_lock(&shrink_list_lock);
5034
5035 rescan:
5036         /* first scan for clean buffers */
5037         list_for_each_entry_safe(dev_priv, next_dev,
5038                                  &shrink_list, mm.shrink_list) {
5039                 struct drm_device *dev = dev_priv->dev;
5040
5041                 if (! mutex_trylock(&dev->struct_mutex))
5042                         continue;
5043
5044                 spin_unlock(&shrink_list_lock);
5045                 i915_gem_retire_requests(dev);
5046
5047                 list_for_each_entry_safe(obj_priv, next_obj,
5048                                          &dev_priv->mm.inactive_list,
5049                                          list) {
5050                         if (i915_gem_object_is_purgeable(obj_priv)) {
5051                                 i915_gem_object_unbind(&obj_priv->base);
5052                                 if (--nr_to_scan <= 0)
5053                                         break;
5054                         }
5055                 }
5056
5057                 spin_lock(&shrink_list_lock);
5058                 mutex_unlock(&dev->struct_mutex);
5059
5060                 would_deadlock = 0;
5061
5062                 if (nr_to_scan <= 0)
5063                         break;
5064         }
5065
5066         /* second pass, evict/count anything still on the inactive list */
5067         list_for_each_entry_safe(dev_priv, next_dev,
5068                                  &shrink_list, mm.shrink_list) {
5069                 struct drm_device *dev = dev_priv->dev;
5070
5071                 if (! mutex_trylock(&dev->struct_mutex))
5072                         continue;
5073
5074                 spin_unlock(&shrink_list_lock);
5075
5076                 list_for_each_entry_safe(obj_priv, next_obj,
5077                                          &dev_priv->mm.inactive_list,
5078                                          list) {
5079                         if (nr_to_scan > 0) {
5080                                 i915_gem_object_unbind(&obj_priv->base);
5081                                 nr_to_scan--;
5082                         } else
5083                                 cnt++;
5084                 }
5085
5086                 spin_lock(&shrink_list_lock);
5087                 mutex_unlock(&dev->struct_mutex);
5088
5089                 would_deadlock = 0;
5090         }
5091
5092         if (nr_to_scan) {
5093                 int active = 0;
5094
5095                 /*
5096                  * We are desperate for pages, so as a last resort, wait
5097                  * for the GPU to finish and discard whatever we can.
5098                  * This has a dramatic impact to reduce the number of
5099                  * OOM-killer events whilst running the GPU aggressively.
5100                  */
5101                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5102                         struct drm_device *dev = dev_priv->dev;
5103
5104                         if (!mutex_trylock(&dev->struct_mutex))
5105                                 continue;
5106
5107                         spin_unlock(&shrink_list_lock);
5108
5109                         if (i915_gpu_is_active(dev)) {
5110                                 i915_gpu_idle(dev);
5111                                 active++;
5112                         }
5113
5114                         spin_lock(&shrink_list_lock);
5115                         mutex_unlock(&dev->struct_mutex);
5116                 }
5117
5118                 if (active)
5119                         goto rescan;
5120         }
5121
5122         spin_unlock(&shrink_list_lock);
5123
5124         if (would_deadlock)
5125                 return -1;
5126         else if (cnt > 0)
5127                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5128         else
5129                 return 0;
5130 }
5131
5132 static struct shrinker shrinker = {
5133         .shrink = i915_gem_shrink,
5134         .seeks = DEFAULT_SEEKS,
5135 };
5136
5137 __init void
5138 i915_gem_shrinker_init(void)
5139 {
5140     register_shrinker(&shrinker);
5141 }
5142
5143 __exit void
5144 i915_gem_shrinker_exit(void)
5145 {
5146     unregister_shrinker(&shrinker);
5147 }