drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/swap.h>
  35 #include <linux/pci.h>
  36
  37 #define I915_GEM_GPU_DOMAINS    (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
  38
  39 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  42 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  43                                              int write);
  44 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  45                                                      uint64_t offset,
  46                                                      uint64_t size);
  47 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  48 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  49 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  50                                            unsigned alignment);
  51 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  52 static int i915_gem_evict_something(struct drm_device *dev, int min_size);
  53 static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
  54 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  55                                 struct drm_i915_gem_pwrite *args,
  56                                 struct drm_file *file_priv);
  57
  58 static LIST_HEAD(shrink_list);
  59 static DEFINE_SPINLOCK(shrink_list_lock);
  60
  61 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  62                      unsigned long end)
  63 {
  64         drm_i915_private_t *dev_priv = dev->dev_private;
  65
  66         if (start >= end ||
  67             (start & (PAGE_SIZE - 1)) != 0 ||
  68             (end & (PAGE_SIZE - 1)) != 0) {
  69                 return -EINVAL;
  70         }
  71
  72         drm_mm_init(&dev_priv->mm.gtt_space, start,
  73                     end - start);
  74
  75         dev->gtt_total = (uint32_t) (end - start);
  76
  77         return 0;
  78 }
  79
  80 int
  81 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  82                     struct drm_file *file_priv)
  83 {
  84         struct drm_i915_gem_init *args = data;
  85         int ret;
  86
  87         mutex_lock(&dev->struct_mutex);
  88         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  89         mutex_unlock(&dev->struct_mutex);
  90
  91         return ret;
  92 }
  93
  94 int
  95 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  96                             struct drm_file *file_priv)
  97 {
  98         struct drm_i915_gem_get_aperture *args = data;
  99
 100         if (!(dev->driver->driver_features & DRIVER_GEM))
 101                 return -ENODEV;
 102
 103         args->aper_size = dev->gtt_total;
 104         args->aper_available_size = (args->aper_size -
 105                                      atomic_read(&dev->pin_memory));
 106
 107         return 0;
 108 }
 109
 110
 111 /**
 112  * Creates a new mm object and returns a handle to it.
 113  */
 114 int
 115 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 116                       struct drm_file *file_priv)
 117 {
 118         struct drm_i915_gem_create *args = data;
 119         struct drm_gem_object *obj;
 120         int ret;
 121         u32 handle;
 122
 123         args->size = roundup(args->size, PAGE_SIZE);
 124
 125         /* Allocate the new object */
 126         obj = drm_gem_object_alloc(dev, args->size);
 127         if (obj == NULL)
 128                 return -ENOMEM;
 129
 130         ret = drm_gem_handle_create(file_priv, obj, &handle);
 131         drm_gem_object_handle_unreference_unlocked(obj);
 132
 133         if (ret)
 134                 return ret;
 135
 136         args->handle = handle;
 137
 138         return 0;
 139 }
 140
 141 static inline int
 142 fast_shmem_read(struct page **pages,
 143                 loff_t page_base, int page_offset,
 144                 char __user *data,
 145                 int length)
 146 {
 147         char __iomem *vaddr;
 148         int unwritten;
 149
 150         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 151         if (vaddr == NULL)
 152                 return -ENOMEM;
 153         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 154         kunmap_atomic(vaddr, KM_USER0);
 155
 156         if (unwritten)
 157                 return -EFAULT;
 158
 159         return 0;
 160 }
 161
 162 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 163 {
 164         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 165         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 166
 167         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 168                 obj_priv->tiling_mode != I915_TILING_NONE;
 169 }
 170
 171 static inline int
 172 slow_shmem_copy(struct page *dst_page,
 173                 int dst_offset,
 174                 struct page *src_page,
 175                 int src_offset,
 176                 int length)
 177 {
 178         char *dst_vaddr, *src_vaddr;
 179
 180         dst_vaddr = kmap_atomic(dst_page, KM_USER0);
 181         if (dst_vaddr == NULL)
 182                 return -ENOMEM;
 183
 184         src_vaddr = kmap_atomic(src_page, KM_USER1);
 185         if (src_vaddr == NULL) {
 186                 kunmap_atomic(dst_vaddr, KM_USER0);
 187                 return -ENOMEM;
 188         }
 189
 190         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 191
 192         kunmap_atomic(src_vaddr, KM_USER1);
 193         kunmap_atomic(dst_vaddr, KM_USER0);
 194
 195         return 0;
 196 }
 197
 198 static inline int
 199 slow_shmem_bit17_copy(struct page *gpu_page,
 200                       int gpu_offset,
 201                       struct page *cpu_page,
 202                       int cpu_offset,
 203                       int length,
 204                       int is_read)
 205 {
 206         char *gpu_vaddr, *cpu_vaddr;
 207
 208         /* Use the unswizzled path if this page isn't affected. */
 209         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 210                 if (is_read)
 211                         return slow_shmem_copy(cpu_page, cpu_offset,
 212                                                gpu_page, gpu_offset, length);
 213                 else
 214                         return slow_shmem_copy(gpu_page, gpu_offset,
 215                                                cpu_page, cpu_offset, length);
 216         }
 217
 218         gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
 219         if (gpu_vaddr == NULL)
 220                 return -ENOMEM;
 221
 222         cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
 223         if (cpu_vaddr == NULL) {
 224                 kunmap_atomic(gpu_vaddr, KM_USER0);
 225                 return -ENOMEM;
 226         }
 227
 228         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 229          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 230          */
 231         while (length > 0) {
 232                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 233                 int this_length = min(cacheline_end - gpu_offset, length);
 234                 int swizzled_gpu_offset = gpu_offset ^ 64;
 235
 236                 if (is_read) {
 237                         memcpy(cpu_vaddr + cpu_offset,
 238                                gpu_vaddr + swizzled_gpu_offset,
 239                                this_length);
 240                 } else {
 241                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 242                                cpu_vaddr + cpu_offset,
 243                                this_length);
 244                 }
 245                 cpu_offset += this_length;
 246                 gpu_offset += this_length;
 247                 length -= this_length;
 248         }
 249
 250         kunmap_atomic(cpu_vaddr, KM_USER1);
 251         kunmap_atomic(gpu_vaddr, KM_USER0);
 252
 253         return 0;
 254 }
 255
 256 /**
 257  * This is the fast shmem pread path, which attempts to copy_from_user directly
 258  * from the backing pages of the object to the user's address space.  On a
 259  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 260  */
 261 static int
 262 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 263                           struct drm_i915_gem_pread *args,
 264                           struct drm_file *file_priv)
 265 {
 266         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 267         ssize_t remain;
 268         loff_t offset, page_base;
 269         char __user *user_data;
 270         int page_offset, page_length;
 271         int ret;
 272
 273         user_data = (char __user *) (uintptr_t) args->data_ptr;
 274         remain = args->size;
 275
 276         mutex_lock(&dev->struct_mutex);
 277
 278         ret = i915_gem_object_get_pages(obj, 0);
 279         if (ret != 0)
 280                 goto fail_unlock;
 281
 282         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 283                                                         args->size);
 284         if (ret != 0)
 285                 goto fail_put_pages;
 286
 287         obj_priv = obj->driver_private;
 288         offset = args->offset;
 289
 290         while (remain > 0) {
 291                 /* Operation in this page
 292                  *
 293                  * page_base = page offset within aperture
 294                  * page_offset = offset within page
 295                  * page_length = bytes to copy for this page
 296                  */
 297                 page_base = (offset & ~(PAGE_SIZE-1));
 298                 page_offset = offset & (PAGE_SIZE-1);
 299                 page_length = remain;
 300                 if ((page_offset + remain) > PAGE_SIZE)
 301                         page_length = PAGE_SIZE - page_offset;
 302
 303                 ret = fast_shmem_read(obj_priv->pages,
 304                                       page_base, page_offset,
 305                                       user_data, page_length);
 306                 if (ret)
 307                         goto fail_put_pages;
 308
 309                 remain -= page_length;
 310                 user_data += page_length;
 311                 offset += page_length;
 312         }
 313
 314 fail_put_pages:
 315         i915_gem_object_put_pages(obj);
 316 fail_unlock:
 317         mutex_unlock(&dev->struct_mutex);
 318
 319         return ret;
 320 }
 321
 322 static int
 323 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 324 {
 325         int ret;
 326
 327         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 328
 329         /* If we've insufficient memory to map in the pages, attempt
 330          * to make some space by throwing out some old buffers.
 331          */
 332         if (ret == -ENOMEM) {
 333                 struct drm_device *dev = obj->dev;
 334
 335                 ret = i915_gem_evict_something(dev, obj->size);
 336                 if (ret)
 337                         return ret;
 338
 339                 ret = i915_gem_object_get_pages(obj, 0);
 340         }
 341
 342         return ret;
 343 }
 344
 345 /**
 346  * This is the fallback shmem pread path, which allocates temporary storage
 347  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 348  * can copy out of the object's backing pages while holding the struct mutex
 349  * and not take page faults.
 350  */
 351 static int
 352 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 353                           struct drm_i915_gem_pread *args,
 354                           struct drm_file *file_priv)
 355 {
 356         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 357         struct mm_struct *mm = current->mm;
 358         struct page **user_pages;
 359         ssize_t remain;
 360         loff_t offset, pinned_pages, i;
 361         loff_t first_data_page, last_data_page, num_pages;
 362         int shmem_page_index, shmem_page_offset;
 363         int data_page_index,  data_page_offset;
 364         int page_length;
 365         int ret;
 366         uint64_t data_ptr = args->data_ptr;
 367         int do_bit17_swizzling;
 368
 369         remain = args->size;
 370
 371         /* Pin the user pages containing the data.  We can't fault while
 372          * holding the struct mutex, yet we want to hold it while
 373          * dereferencing the user data.
 374          */
 375         first_data_page = data_ptr / PAGE_SIZE;
 376         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 377         num_pages = last_data_page - first_data_page + 1;
 378
 379         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 380         if (user_pages == NULL)
 381                 return -ENOMEM;
 382
 383         down_read(&mm->mmap_sem);
 384         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 385                                       num_pages, 1, 0, user_pages, NULL);
 386         up_read(&mm->mmap_sem);
 387         if (pinned_pages < num_pages) {
 388                 ret = -EFAULT;
 389                 goto fail_put_user_pages;
 390         }
 391
 392         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 393
 394         mutex_lock(&dev->struct_mutex);
 395
 396         ret = i915_gem_object_get_pages_or_evict(obj);
 397         if (ret)
 398                 goto fail_unlock;
 399
 400         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 401                                                         args->size);
 402         if (ret != 0)
 403                 goto fail_put_pages;
 404
 405         obj_priv = obj->driver_private;
 406         offset = args->offset;
 407
 408         while (remain > 0) {
 409                 /* Operation in this page
 410                  *
 411                  * shmem_page_index = page number within shmem file
 412                  * shmem_page_offset = offset within page in shmem file
 413                  * data_page_index = page number in get_user_pages return
 414                  * data_page_offset = offset with data_page_index page.
 415                  * page_length = bytes to copy for this page
 416                  */
 417                 shmem_page_index = offset / PAGE_SIZE;
 418                 shmem_page_offset = offset & ~PAGE_MASK;
 419                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 420                 data_page_offset = data_ptr & ~PAGE_MASK;
 421
 422                 page_length = remain;
 423                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 424                         page_length = PAGE_SIZE - shmem_page_offset;
 425                 if ((data_page_offset + page_length) > PAGE_SIZE)
 426                         page_length = PAGE_SIZE - data_page_offset;
 427
 428                 if (do_bit17_swizzling) {
 429                         ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 430                                                     shmem_page_offset,
 431                                                     user_pages[data_page_index],
 432                                                     data_page_offset,
 433                                                     page_length,
 434                                                     1);
 435                 } else {
 436                         ret = slow_shmem_copy(user_pages[data_page_index],
 437                                               data_page_offset,
 438                                               obj_priv->pages[shmem_page_index],
 439                                               shmem_page_offset,
 440                                               page_length);
 441                 }
 442                 if (ret)
 443                         goto fail_put_pages;
 444
 445                 remain -= page_length;
 446                 data_ptr += page_length;
 447                 offset += page_length;
 448         }
 449
 450 fail_put_pages:
 451         i915_gem_object_put_pages(obj);
 452 fail_unlock:
 453         mutex_unlock(&dev->struct_mutex);
 454 fail_put_user_pages:
 455         for (i = 0; i < pinned_pages; i++) {
 456                 SetPageDirty(user_pages[i]);
 457                 page_cache_release(user_pages[i]);
 458         }
 459         drm_free_large(user_pages);
 460
 461         return ret;
 462 }
 463
 464 /**
 465  * Reads data from the object referenced by handle.
 466  *
 467  * On error, the contents of *data are undefined.
 468  */
 469 int
 470 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 471                      struct drm_file *file_priv)
 472 {
 473         struct drm_i915_gem_pread *args = data;
 474         struct drm_gem_object *obj;
 475         struct drm_i915_gem_object *obj_priv;
 476         int ret;
 477
 478         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 479         if (obj == NULL)
 480                 return -EBADF;
 481         obj_priv = obj->driver_private;
 482
 483         /* Bounds check source.
 484          *
 485          * XXX: This could use review for overflow issues...
 486          */
 487         if (args->offset > obj->size || args->size > obj->size ||
 488             args->offset + args->size > obj->size) {
 489                 drm_gem_object_unreference_unlocked(obj);
 490                 return -EINVAL;
 491         }
 492
 493         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 494                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 495         } else {
 496                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 497                 if (ret != 0)
 498                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 499                                                         file_priv);
 500         }
 501
 502         drm_gem_object_unreference_unlocked(obj);
 503
 504         return ret;
 505 }
 506
 507 /* This is the fast write path which cannot handle
 508  * page faults in the source data
 509  */
 510
 511 static inline int
 512 fast_user_write(struct io_mapping *mapping,
 513                 loff_t page_base, int page_offset,
 514                 char __user *user_data,
 515                 int length)
 516 {
 517         char *vaddr_atomic;
 518         unsigned long unwritten;
 519
 520         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 521         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 522                                                       user_data, length);
 523         io_mapping_unmap_atomic(vaddr_atomic);
 524         if (unwritten)
 525                 return -EFAULT;
 526         return 0;
 527 }
 528
 529 /* Here's the write path which can sleep for
 530  * page faults
 531  */
 532
 533 static inline int
 534 slow_kernel_write(struct io_mapping *mapping,
 535                   loff_t gtt_base, int gtt_offset,
 536                   struct page *user_page, int user_offset,
 537                   int length)
 538 {
 539         char *src_vaddr, *dst_vaddr;
 540         unsigned long unwritten;
 541
 542         dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
 543         src_vaddr = kmap_atomic(user_page, KM_USER1);
 544         unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
 545                                                       src_vaddr + user_offset,
 546                                                       length);
 547         kunmap_atomic(src_vaddr, KM_USER1);
 548         io_mapping_unmap_atomic(dst_vaddr);
 549         if (unwritten)
 550                 return -EFAULT;
 551         return 0;
 552 }
 553
 554 static inline int
 555 fast_shmem_write(struct page **pages,
 556                  loff_t page_base, int page_offset,
 557                  char __user *data,
 558                  int length)
 559 {
 560         char __iomem *vaddr;
 561         unsigned long unwritten;
 562
 563         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 564         if (vaddr == NULL)
 565                 return -ENOMEM;
 566         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 567         kunmap_atomic(vaddr, KM_USER0);
 568
 569         if (unwritten)
 570                 return -EFAULT;
 571         return 0;
 572 }
 573
 574 /**
 575  * This is the fast pwrite path, where we copy the data directly from the
 576  * user into the GTT, uncached.
 577  */
 578 static int
 579 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 580                          struct drm_i915_gem_pwrite *args,
 581                          struct drm_file *file_priv)
 582 {
 583         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 584         drm_i915_private_t *dev_priv = dev->dev_private;
 585         ssize_t remain;
 586         loff_t offset, page_base;
 587         char __user *user_data;
 588         int page_offset, page_length;
 589         int ret;
 590
 591         user_data = (char __user *) (uintptr_t) args->data_ptr;
 592         remain = args->size;
 593         if (!access_ok(VERIFY_READ, user_data, remain))
 594                 return -EFAULT;
 595
 596
 597         mutex_lock(&dev->struct_mutex);
 598         ret = i915_gem_object_pin(obj, 0);
 599         if (ret) {
 600                 mutex_unlock(&dev->struct_mutex);
 601                 return ret;
 602         }
 603         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 604         if (ret)
 605                 goto fail;
 606
 607         obj_priv = obj->driver_private;
 608         offset = obj_priv->gtt_offset + args->offset;
 609
 610         while (remain > 0) {
 611                 /* Operation in this page
 612                  *
 613                  * page_base = page offset within aperture
 614                  * page_offset = offset within page
 615                  * page_length = bytes to copy for this page
 616                  */
 617                 page_base = (offset & ~(PAGE_SIZE-1));
 618                 page_offset = offset & (PAGE_SIZE-1);
 619                 page_length = remain;
 620                 if ((page_offset + remain) > PAGE_SIZE)
 621                         page_length = PAGE_SIZE - page_offset;
 622
 623                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 624                                        page_offset, user_data, page_length);
 625
 626                 /* If we get a fault while copying data, then (presumably) our
 627                  * source page isn't available.  Return the error and we'll
 628                  * retry in the slow path.
 629                  */
 630                 if (ret)
 631                         goto fail;
 632
 633                 remain -= page_length;
 634                 user_data += page_length;
 635                 offset += page_length;
 636         }
 637
 638 fail:
 639         i915_gem_object_unpin(obj);
 640         mutex_unlock(&dev->struct_mutex);
 641
 642         return ret;
 643 }
 644
 645 /**
 646  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 647  * the memory and maps it using kmap_atomic for copying.
 648  *
 649  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 650  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 651  */
 652 static int
 653 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 654                          struct drm_i915_gem_pwrite *args,
 655                          struct drm_file *file_priv)
 656 {
 657         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 658         drm_i915_private_t *dev_priv = dev->dev_private;
 659         ssize_t remain;
 660         loff_t gtt_page_base, offset;
 661         loff_t first_data_page, last_data_page, num_pages;
 662         loff_t pinned_pages, i;
 663         struct page **user_pages;
 664         struct mm_struct *mm = current->mm;
 665         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 666         int ret;
 667         uint64_t data_ptr = args->data_ptr;
 668
 669         remain = args->size;
 670
 671         /* Pin the user pages containing the data.  We can't fault while
 672          * holding the struct mutex, and all of the pwrite implementations
 673          * want to hold it while dereferencing the user data.
 674          */
 675         first_data_page = data_ptr / PAGE_SIZE;
 676         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 677         num_pages = last_data_page - first_data_page + 1;
 678
 679         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 680         if (user_pages == NULL)
 681                 return -ENOMEM;
 682
 683         down_read(&mm->mmap_sem);
 684         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 685                                       num_pages, 0, 0, user_pages, NULL);
 686         up_read(&mm->mmap_sem);
 687         if (pinned_pages < num_pages) {
 688                 ret = -EFAULT;
 689                 goto out_unpin_pages;
 690         }
 691
 692         mutex_lock(&dev->struct_mutex);
 693         ret = i915_gem_object_pin(obj, 0);
 694         if (ret)
 695                 goto out_unlock;
 696
 697         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 698         if (ret)
 699                 goto out_unpin_object;
 700
 701         obj_priv = obj->driver_private;
 702         offset = obj_priv->gtt_offset + args->offset;
 703
 704         while (remain > 0) {
 705                 /* Operation in this page
 706                  *
 707                  * gtt_page_base = page offset within aperture
 708                  * gtt_page_offset = offset within page in aperture
 709                  * data_page_index = page number in get_user_pages return
 710                  * data_page_offset = offset with data_page_index page.
 711                  * page_length = bytes to copy for this page
 712                  */
 713                 gtt_page_base = offset & PAGE_MASK;
 714                 gtt_page_offset = offset & ~PAGE_MASK;
 715                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 716                 data_page_offset = data_ptr & ~PAGE_MASK;
 717
 718                 page_length = remain;
 719                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 720                         page_length = PAGE_SIZE - gtt_page_offset;
 721                 if ((data_page_offset + page_length) > PAGE_SIZE)
 722                         page_length = PAGE_SIZE - data_page_offset;
 723
 724                 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
 725                                         gtt_page_base, gtt_page_offset,
 726                                         user_pages[data_page_index],
 727                                         data_page_offset,
 728                                         page_length);
 729
 730                 /* If we get a fault while copying data, then (presumably) our
 731                  * source page isn't available.  Return the error and we'll
 732                  * retry in the slow path.
 733                  */
 734                 if (ret)
 735                         goto out_unpin_object;
 736
 737                 remain -= page_length;
 738                 offset += page_length;
 739                 data_ptr += page_length;
 740         }
 741
 742 out_unpin_object:
 743         i915_gem_object_unpin(obj);
 744 out_unlock:
 745         mutex_unlock(&dev->struct_mutex);
 746 out_unpin_pages:
 747         for (i = 0; i < pinned_pages; i++)
 748                 page_cache_release(user_pages[i]);
 749         drm_free_large(user_pages);
 750
 751         return ret;
 752 }
 753
 754 /**
 755  * This is the fast shmem pwrite path, which attempts to directly
 756  * copy_from_user into the kmapped pages backing the object.
 757  */
 758 static int
 759 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 760                            struct drm_i915_gem_pwrite *args,
 761                            struct drm_file *file_priv)
 762 {
 763         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 764         ssize_t remain;
 765         loff_t offset, page_base;
 766         char __user *user_data;
 767         int page_offset, page_length;
 768         int ret;
 769
 770         user_data = (char __user *) (uintptr_t) args->data_ptr;
 771         remain = args->size;
 772
 773         mutex_lock(&dev->struct_mutex);
 774
 775         ret = i915_gem_object_get_pages(obj, 0);
 776         if (ret != 0)
 777                 goto fail_unlock;
 778
 779         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 780         if (ret != 0)
 781                 goto fail_put_pages;
 782
 783         obj_priv = obj->driver_private;
 784         offset = args->offset;
 785         obj_priv->dirty = 1;
 786
 787         while (remain > 0) {
 788                 /* Operation in this page
 789                  *
 790                  * page_base = page offset within aperture
 791                  * page_offset = offset within page
 792                  * page_length = bytes to copy for this page
 793                  */
 794                 page_base = (offset & ~(PAGE_SIZE-1));
 795                 page_offset = offset & (PAGE_SIZE-1);
 796                 page_length = remain;
 797                 if ((page_offset + remain) > PAGE_SIZE)
 798                         page_length = PAGE_SIZE - page_offset;
 799
 800                 ret = fast_shmem_write(obj_priv->pages,
 801                                        page_base, page_offset,
 802                                        user_data, page_length);
 803                 if (ret)
 804                         goto fail_put_pages;
 805
 806                 remain -= page_length;
 807                 user_data += page_length;
 808                 offset += page_length;
 809         }
 810
 811 fail_put_pages:
 812         i915_gem_object_put_pages(obj);
 813 fail_unlock:
 814         mutex_unlock(&dev->struct_mutex);
 815
 816         return ret;
 817 }
 818
 819 /**
 820  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 821  * the memory and maps it using kmap_atomic for copying.
 822  *
 823  * This avoids taking mmap_sem for faulting on the user's address while the
 824  * struct_mutex is held.
 825  */
 826 static int
 827 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 828                            struct drm_i915_gem_pwrite *args,
 829                            struct drm_file *file_priv)
 830 {
 831         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 832         struct mm_struct *mm = current->mm;
 833         struct page **user_pages;
 834         ssize_t remain;
 835         loff_t offset, pinned_pages, i;
 836         loff_t first_data_page, last_data_page, num_pages;
 837         int shmem_page_index, shmem_page_offset;
 838         int data_page_index,  data_page_offset;
 839         int page_length;
 840         int ret;
 841         uint64_t data_ptr = args->data_ptr;
 842         int do_bit17_swizzling;
 843
 844         remain = args->size;
 845
 846         /* Pin the user pages containing the data.  We can't fault while
 847          * holding the struct mutex, and all of the pwrite implementations
 848          * want to hold it while dereferencing the user data.
 849          */
 850         first_data_page = data_ptr / PAGE_SIZE;
 851         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 852         num_pages = last_data_page - first_data_page + 1;
 853
 854         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 855         if (user_pages == NULL)
 856                 return -ENOMEM;
 857
 858         down_read(&mm->mmap_sem);
 859         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 860                                       num_pages, 0, 0, user_pages, NULL);
 861         up_read(&mm->mmap_sem);
 862         if (pinned_pages < num_pages) {
 863                 ret = -EFAULT;
 864                 goto fail_put_user_pages;
 865         }
 866
 867         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 868
 869         mutex_lock(&dev->struct_mutex);
 870
 871         ret = i915_gem_object_get_pages_or_evict(obj);
 872         if (ret)
 873                 goto fail_unlock;
 874
 875         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 876         if (ret != 0)
 877                 goto fail_put_pages;
 878
 879         obj_priv = obj->driver_private;
 880         offset = args->offset;
 881         obj_priv->dirty = 1;
 882
 883         while (remain > 0) {
 884                 /* Operation in this page
 885                  *
 886                  * shmem_page_index = page number within shmem file
 887                  * shmem_page_offset = offset within page in shmem file
 888                  * data_page_index = page number in get_user_pages return
 889                  * data_page_offset = offset with data_page_index page.
 890                  * page_length = bytes to copy for this page
 891                  */
 892                 shmem_page_index = offset / PAGE_SIZE;
 893                 shmem_page_offset = offset & ~PAGE_MASK;
 894                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 895                 data_page_offset = data_ptr & ~PAGE_MASK;
 896
 897                 page_length = remain;
 898                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 899                         page_length = PAGE_SIZE - shmem_page_offset;
 900                 if ((data_page_offset + page_length) > PAGE_SIZE)
 901                         page_length = PAGE_SIZE - data_page_offset;
 902
 903                 if (do_bit17_swizzling) {
 904                         ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 905                                                     shmem_page_offset,
 906                                                     user_pages[data_page_index],
 907                                                     data_page_offset,
 908                                                     page_length,
 909                                                     0);
 910                 } else {
 911                         ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
 912                                               shmem_page_offset,
 913                                               user_pages[data_page_index],
 914                                               data_page_offset,
 915                                               page_length);
 916                 }
 917                 if (ret)
 918                         goto fail_put_pages;
 919
 920                 remain -= page_length;
 921                 data_ptr += page_length;
 922                 offset += page_length;
 923         }
 924
 925 fail_put_pages:
 926         i915_gem_object_put_pages(obj);
 927 fail_unlock:
 928         mutex_unlock(&dev->struct_mutex);
 929 fail_put_user_pages:
 930         for (i = 0; i < pinned_pages; i++)
 931                 page_cache_release(user_pages[i]);
 932         drm_free_large(user_pages);
 933
 934         return ret;
 935 }
 936
 937 /**
 938  * Writes data to the object referenced by handle.
 939  *
 940  * On error, the contents of the buffer that were to be modified are undefined.
 941  */
 942 int
 943 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 944                       struct drm_file *file_priv)
 945 {
 946         struct drm_i915_gem_pwrite *args = data;
 947         struct drm_gem_object *obj;
 948         struct drm_i915_gem_object *obj_priv;
 949         int ret = 0;
 950
 951         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 952         if (obj == NULL)
 953                 return -EBADF;
 954         obj_priv = obj->driver_private;
 955
 956         /* Bounds check destination.
 957          *
 958          * XXX: This could use review for overflow issues...
 959          */
 960         if (args->offset > obj->size || args->size > obj->size ||
 961             args->offset + args->size > obj->size) {
 962                 drm_gem_object_unreference_unlocked(obj);
 963                 return -EINVAL;
 964         }
 965
 966         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 967          * it would end up going through the fenced access, and we'll get
 968          * different detiling behavior between reading and writing.
 969          * pread/pwrite currently are reading and writing from the CPU
 970          * perspective, requiring manual detiling by the client.
 971          */
 972         if (obj_priv->phys_obj)
 973                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 974         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 975                  dev->gtt_total != 0) {
 976                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 977                 if (ret == -EFAULT) {
 978                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 979                                                        file_priv);
 980                 }
 981         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 982                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 983         } else {
 984                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 985                 if (ret == -EFAULT) {
 986                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 987                                                          file_priv);
 988                 }
 989         }
 990
 991 #if WATCH_PWRITE
 992         if (ret)
 993                 DRM_INFO("pwrite failed %d\n", ret);
 994 #endif
 995
 996         drm_gem_object_unreference_unlocked(obj);
 997
 998         return ret;
 999 }
1000
1001 /**
1002  * Called when user space prepares to use an object with the CPU, either
1003  * through the mmap ioctl's mapping or a GTT mapping.
1004  */
1005 int
1006 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1007                           struct drm_file *file_priv)
1008 {
1009         struct drm_i915_private *dev_priv = dev->dev_private;
1010         struct drm_i915_gem_set_domain *args = data;
1011         struct drm_gem_object *obj;
1012         struct drm_i915_gem_object *obj_priv;
1013         uint32_t read_domains = args->read_domains;
1014         uint32_t write_domain = args->write_domain;
1015         int ret;
1016
1017         if (!(dev->driver->driver_features & DRIVER_GEM))
1018                 return -ENODEV;
1019
1020         /* Only handle setting domains to types used by the CPU. */
1021         if (write_domain & I915_GEM_GPU_DOMAINS)
1022                 return -EINVAL;
1023
1024         if (read_domains & I915_GEM_GPU_DOMAINS)
1025                 return -EINVAL;
1026
1027         /* Having something in the write domain implies it's in the read
1028          * domain, and only that read domain.  Enforce that in the request.
1029          */
1030         if (write_domain != 0 && read_domains != write_domain)
1031                 return -EINVAL;
1032
1033         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1034         if (obj == NULL)
1035                 return -EBADF;
1036         obj_priv = obj->driver_private;
1037
1038         mutex_lock(&dev->struct_mutex);
1039
1040         intel_mark_busy(dev, obj);
1041
1042 #if WATCH_BUF
1043         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1044                  obj, obj->size, read_domains, write_domain);
1045 #endif
1046         if (read_domains & I915_GEM_DOMAIN_GTT) {
1047                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1048
1049                 /* Update the LRU on the fence for the CPU access that's
1050                  * about to occur.
1051                  */
1052                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1053                         list_move_tail(&obj_priv->fence_list,
1054                                        &dev_priv->mm.fence_list);
1055                 }
1056
1057                 /* Silently promote "you're not bound, there was nothing to do"
1058                  * to success, since the client was just asking us to
1059                  * make sure everything was done.
1060                  */
1061                 if (ret == -EINVAL)
1062                         ret = 0;
1063         } else {
1064                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1065         }
1066
1067         drm_gem_object_unreference(obj);
1068         mutex_unlock(&dev->struct_mutex);
1069         return ret;
1070 }
1071
1072 /**
1073  * Called when user space has done writes to this buffer
1074  */
1075 int
1076 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1077                       struct drm_file *file_priv)
1078 {
1079         struct drm_i915_gem_sw_finish *args = data;
1080         struct drm_gem_object *obj;
1081         struct drm_i915_gem_object *obj_priv;
1082         int ret = 0;
1083
1084         if (!(dev->driver->driver_features & DRIVER_GEM))
1085                 return -ENODEV;
1086
1087         mutex_lock(&dev->struct_mutex);
1088         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1089         if (obj == NULL) {
1090                 mutex_unlock(&dev->struct_mutex);
1091                 return -EBADF;
1092         }
1093
1094 #if WATCH_BUF
1095         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1096                  __func__, args->handle, obj, obj->size);
1097 #endif
1098         obj_priv = obj->driver_private;
1099
1100         /* Pinned buffers may be scanout, so flush the cache */
1101         if (obj_priv->pin_count)
1102                 i915_gem_object_flush_cpu_write_domain(obj);
1103
1104         drm_gem_object_unreference(obj);
1105         mutex_unlock(&dev->struct_mutex);
1106         return ret;
1107 }
1108
1109 /**
1110  * Maps the contents of an object, returning the address it is mapped
1111  * into.
1112  *
1113  * While the mapping holds a reference on the contents of the object, it doesn't
1114  * imply a ref on the object itself.
1115  */
1116 int
1117 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1118                    struct drm_file *file_priv)
1119 {
1120         struct drm_i915_gem_mmap *args = data;
1121         struct drm_gem_object *obj;
1122         loff_t offset;
1123         unsigned long addr;
1124
1125         if (!(dev->driver->driver_features & DRIVER_GEM))
1126                 return -ENODEV;
1127
1128         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1129         if (obj == NULL)
1130                 return -EBADF;
1131
1132         offset = args->offset;
1133
1134         down_write(&current->mm->mmap_sem);
1135         addr = do_mmap(obj->filp, 0, args->size,
1136                        PROT_READ | PROT_WRITE, MAP_SHARED,
1137                        args->offset);
1138         up_write(&current->mm->mmap_sem);
1139         drm_gem_object_unreference_unlocked(obj);
1140         if (IS_ERR((void *)addr))
1141                 return addr;
1142
1143         args->addr_ptr = (uint64_t) addr;
1144
1145         return 0;
1146 }
1147
1148 /**
1149  * i915_gem_fault - fault a page into the GTT
1150  * vma: VMA in question
1151  * vmf: fault info
1152  *
1153  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1154  * from userspace.  The fault handler takes care of binding the object to
1155  * the GTT (if needed), allocating and programming a fence register (again,
1156  * only if needed based on whether the old reg is still valid or the object
1157  * is tiled) and inserting a new PTE into the faulting process.
1158  *
1159  * Note that the faulting process may involve evicting existing objects
1160  * from the GTT and/or fence registers to make room.  So performance may
1161  * suffer if the GTT working set is large or there are few fence registers
1162  * left.
1163  */
1164 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1165 {
1166         struct drm_gem_object *obj = vma->vm_private_data;
1167         struct drm_device *dev = obj->dev;
1168         struct drm_i915_private *dev_priv = dev->dev_private;
1169         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1170         pgoff_t page_offset;
1171         unsigned long pfn;
1172         int ret = 0;
1173         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1174
1175         /* We don't use vmf->pgoff since that has the fake offset */
1176         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1177                 PAGE_SHIFT;
1178
1179         /* Now bind it into the GTT if needed */
1180         mutex_lock(&dev->struct_mutex);
1181         if (!obj_priv->gtt_space) {
1182                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1183                 if (ret)
1184                         goto unlock;
1185
1186                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1187
1188                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1189                 if (ret)
1190                         goto unlock;
1191         }
1192
1193         /* Need a new fence register? */
1194         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1195                 ret = i915_gem_object_get_fence_reg(obj);
1196                 if (ret)
1197                         goto unlock;
1198         }
1199
1200         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1201                 page_offset;
1202
1203         /* Finally, remap it using the new GTT offset */
1204         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1205 unlock:
1206         mutex_unlock(&dev->struct_mutex);
1207
1208         switch (ret) {
1209         case 0:
1210         case -ERESTARTSYS:
1211                 return VM_FAULT_NOPAGE;
1212         case -ENOMEM:
1213         case -EAGAIN:
1214                 return VM_FAULT_OOM;
1215         default:
1216                 return VM_FAULT_SIGBUS;
1217         }
1218 }
1219
1220 /**
1221  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1222  * @obj: obj in question
1223  *
1224  * GEM memory mapping works by handing back to userspace a fake mmap offset
1225  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1226  * up the object based on the offset and sets up the various memory mapping
1227  * structures.
1228  *
1229  * This routine allocates and attaches a fake offset for @obj.
1230  */
1231 static int
1232 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1233 {
1234         struct drm_device *dev = obj->dev;
1235         struct drm_gem_mm *mm = dev->mm_private;
1236         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1237         struct drm_map_list *list;
1238         struct drm_local_map *map;
1239         int ret = 0;
1240
1241         /* Set the object up for mmap'ing */
1242         list = &obj->map_list;
1243         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1244         if (!list->map)
1245                 return -ENOMEM;
1246
1247         map = list->map;
1248         map->type = _DRM_GEM;
1249         map->size = obj->size;
1250         map->handle = obj;
1251
1252         /* Get a DRM GEM mmap offset allocated... */
1253         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1254                                                     obj->size / PAGE_SIZE, 0, 0);
1255         if (!list->file_offset_node) {
1256                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1257                 ret = -ENOMEM;
1258                 goto out_free_list;
1259         }
1260
1261         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1262                                                   obj->size / PAGE_SIZE, 0);
1263         if (!list->file_offset_node) {
1264                 ret = -ENOMEM;
1265                 goto out_free_list;
1266         }
1267
1268         list->hash.key = list->file_offset_node->start;
1269         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1270                 DRM_ERROR("failed to add to map hash\n");
1271                 ret = -ENOMEM;
1272                 goto out_free_mm;
1273         }
1274
1275         /* By now we should be all set, any drm_mmap request on the offset
1276          * below will get to our mmap & fault handler */
1277         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1278
1279         return 0;
1280
1281 out_free_mm:
1282         drm_mm_put_block(list->file_offset_node);
1283 out_free_list:
1284         kfree(list->map);
1285
1286         return ret;
1287 }
1288
1289 /**
1290  * i915_gem_release_mmap - remove physical page mappings
1291  * @obj: obj in question
1292  *
1293  * Preserve the reservation of the mmapping with the DRM core code, but
1294  * relinquish ownership of the pages back to the system.
1295  *
1296  * It is vital that we remove the page mapping if we have mapped a tiled
1297  * object through the GTT and then lose the fence register due to
1298  * resource pressure. Similarly if the object has been moved out of the
1299  * aperture, than pages mapped into userspace must be revoked. Removing the
1300  * mapping will then trigger a page fault on the next user access, allowing
1301  * fixup by i915_gem_fault().
1302  */
1303 void
1304 i915_gem_release_mmap(struct drm_gem_object *obj)
1305 {
1306         struct drm_device *dev = obj->dev;
1307         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1308
1309         if (dev->dev_mapping)
1310                 unmap_mapping_range(dev->dev_mapping,
1311                                     obj_priv->mmap_offset, obj->size, 1);
1312 }
1313
1314 static void
1315 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1316 {
1317         struct drm_device *dev = obj->dev;
1318         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1319         struct drm_gem_mm *mm = dev->mm_private;
1320         struct drm_map_list *list;
1321
1322         list = &obj->map_list;
1323         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1324
1325         if (list->file_offset_node) {
1326                 drm_mm_put_block(list->file_offset_node);
1327                 list->file_offset_node = NULL;
1328         }
1329
1330         if (list->map) {
1331                 kfree(list->map);
1332                 list->map = NULL;
1333         }
1334
1335         obj_priv->mmap_offset = 0;
1336 }
1337
1338 /**
1339  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1340  * @obj: object to check
1341  *
1342  * Return the required GTT alignment for an object, taking into account
1343  * potential fence register mapping if needed.
1344  */
1345 static uint32_t
1346 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1347 {
1348         struct drm_device *dev = obj->dev;
1349         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1350         int start, i;
1351
1352         /*
1353          * Minimum alignment is 4k (GTT page size), but might be greater
1354          * if a fence register is needed for the object.
1355          */
1356         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1357                 return 4096;
1358
1359         /*
1360          * Previous chips need to be aligned to the size of the smallest
1361          * fence register that can contain the object.
1362          */
1363         if (IS_I9XX(dev))
1364                 start = 1024*1024;
1365         else
1366                 start = 512*1024;
1367
1368         for (i = start; i < obj->size; i <<= 1)
1369                 ;
1370
1371         return i;
1372 }
1373
1374 /**
1375  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1376  * @dev: DRM device
1377  * @data: GTT mapping ioctl data
1378  * @file_priv: GEM object info
1379  *
1380  * Simply returns the fake offset to userspace so it can mmap it.
1381  * The mmap call will end up in drm_gem_mmap(), which will set things
1382  * up so we can get faults in the handler above.
1383  *
1384  * The fault handler will take care of binding the object into the GTT
1385  * (since it may have been evicted to make room for something), allocating
1386  * a fence register, and mapping the appropriate aperture address into
1387  * userspace.
1388  */
1389 int
1390 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1391                         struct drm_file *file_priv)
1392 {
1393         struct drm_i915_gem_mmap_gtt *args = data;
1394         struct drm_i915_private *dev_priv = dev->dev_private;
1395         struct drm_gem_object *obj;
1396         struct drm_i915_gem_object *obj_priv;
1397         int ret;
1398
1399         if (!(dev->driver->driver_features & DRIVER_GEM))
1400                 return -ENODEV;
1401
1402         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1403         if (obj == NULL)
1404                 return -EBADF;
1405
1406         mutex_lock(&dev->struct_mutex);
1407
1408         obj_priv = obj->driver_private;
1409
1410         if (obj_priv->madv != I915_MADV_WILLNEED) {
1411                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1412                 drm_gem_object_unreference(obj);
1413                 mutex_unlock(&dev->struct_mutex);
1414                 return -EINVAL;
1415         }
1416
1417
1418         if (!obj_priv->mmap_offset) {
1419                 ret = i915_gem_create_mmap_offset(obj);
1420                 if (ret) {
1421                         drm_gem_object_unreference(obj);
1422                         mutex_unlock(&dev->struct_mutex);
1423                         return ret;
1424                 }
1425         }
1426
1427         args->offset = obj_priv->mmap_offset;
1428
1429         /*
1430          * Pull it into the GTT so that we have a page list (makes the
1431          * initial fault faster and any subsequent flushing possible).
1432          */
1433         if (!obj_priv->agp_mem) {
1434                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1435                 if (ret) {
1436                         drm_gem_object_unreference(obj);
1437                         mutex_unlock(&dev->struct_mutex);
1438                         return ret;
1439                 }
1440                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1441         }
1442
1443         drm_gem_object_unreference(obj);
1444         mutex_unlock(&dev->struct_mutex);
1445
1446         return 0;
1447 }
1448
1449 void
1450 i915_gem_object_put_pages(struct drm_gem_object *obj)
1451 {
1452         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1453         int page_count = obj->size / PAGE_SIZE;
1454         int i;
1455
1456         BUG_ON(obj_priv->pages_refcount == 0);
1457         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1458
1459         if (--obj_priv->pages_refcount != 0)
1460                 return;
1461
1462         if (obj_priv->tiling_mode != I915_TILING_NONE)
1463                 i915_gem_object_save_bit_17_swizzle(obj);
1464
1465         if (obj_priv->madv == I915_MADV_DONTNEED)
1466                 obj_priv->dirty = 0;
1467
1468         for (i = 0; i < page_count; i++) {
1469                 if (obj_priv->pages[i] == NULL)
1470                         break;
1471
1472                 if (obj_priv->dirty)
1473                         set_page_dirty(obj_priv->pages[i]);
1474
1475                 if (obj_priv->madv == I915_MADV_WILLNEED)
1476                         mark_page_accessed(obj_priv->pages[i]);
1477
1478                 page_cache_release(obj_priv->pages[i]);
1479         }
1480         obj_priv->dirty = 0;
1481
1482         drm_free_large(obj_priv->pages);
1483         obj_priv->pages = NULL;
1484 }
1485
1486 static void
1487 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
1488 {
1489         struct drm_device *dev = obj->dev;
1490         drm_i915_private_t *dev_priv = dev->dev_private;
1491         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1492
1493         /* Add a reference if we're newly entering the active list. */
1494         if (!obj_priv->active) {
1495                 drm_gem_object_reference(obj);
1496                 obj_priv->active = 1;
1497         }
1498         /* Move from whatever list we were on to the tail of execution. */
1499         spin_lock(&dev_priv->mm.active_list_lock);
1500         list_move_tail(&obj_priv->list,
1501                        &dev_priv->mm.active_list);
1502         spin_unlock(&dev_priv->mm.active_list_lock);
1503         obj_priv->last_rendering_seqno = seqno;
1504 }
1505
1506 static void
1507 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1508 {
1509         struct drm_device *dev = obj->dev;
1510         drm_i915_private_t *dev_priv = dev->dev_private;
1511         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1512
1513         BUG_ON(!obj_priv->active);
1514         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1515         obj_priv->last_rendering_seqno = 0;
1516 }
1517
1518 /* Immediately discard the backing storage */
1519 static void
1520 i915_gem_object_truncate(struct drm_gem_object *obj)
1521 {
1522         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1523         struct inode *inode;
1524
1525         inode = obj->filp->f_path.dentry->d_inode;
1526         if (inode->i_op->truncate)
1527                 inode->i_op->truncate (inode);
1528
1529         obj_priv->madv = __I915_MADV_PURGED;
1530 }
1531
1532 static inline int
1533 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1534 {
1535         return obj_priv->madv == I915_MADV_DONTNEED;
1536 }
1537
1538 static void
1539 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1540 {
1541         struct drm_device *dev = obj->dev;
1542         drm_i915_private_t *dev_priv = dev->dev_private;
1543         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1544
1545         i915_verify_inactive(dev, __FILE__, __LINE__);
1546         if (obj_priv->pin_count != 0)
1547                 list_del_init(&obj_priv->list);
1548         else
1549                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1550
1551         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1552
1553         obj_priv->last_rendering_seqno = 0;
1554         if (obj_priv->active) {
1555                 obj_priv->active = 0;
1556                 drm_gem_object_unreference(obj);
1557         }
1558         i915_verify_inactive(dev, __FILE__, __LINE__);
1559 }
1560
1561 /**
1562  * Creates a new sequence number, emitting a write of it to the status page
1563  * plus an interrupt, which will trigger i915_user_interrupt_handler.
1564  *
1565  * Must be called with struct_lock held.
1566  *
1567  * Returned sequence numbers are nonzero on success.
1568  */
1569 uint32_t
1570 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1571                  uint32_t flush_domains)
1572 {
1573         drm_i915_private_t *dev_priv = dev->dev_private;
1574         struct drm_i915_file_private *i915_file_priv = NULL;
1575         struct drm_i915_gem_request *request;
1576         uint32_t seqno;
1577         int was_empty;
1578         RING_LOCALS;
1579
1580         if (file_priv != NULL)
1581                 i915_file_priv = file_priv->driver_priv;
1582
1583         request = kzalloc(sizeof(*request), GFP_KERNEL);
1584         if (request == NULL)
1585                 return 0;
1586
1587         /* Grab the seqno we're going to make this request be, and bump the
1588          * next (skipping 0 so it can be the reserved no-seqno value).
1589          */
1590         seqno = dev_priv->mm.next_gem_seqno;
1591         dev_priv->mm.next_gem_seqno++;
1592         if (dev_priv->mm.next_gem_seqno == 0)
1593                 dev_priv->mm.next_gem_seqno++;
1594
1595         BEGIN_LP_RING(4);
1596         OUT_RING(MI_STORE_DWORD_INDEX);
1597         OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1598         OUT_RING(seqno);
1599
1600         OUT_RING(MI_USER_INTERRUPT);
1601         ADVANCE_LP_RING();
1602
1603         DRM_DEBUG_DRIVER("%d\n", seqno);
1604
1605         request->seqno = seqno;
1606         request->emitted_jiffies = jiffies;
1607         was_empty = list_empty(&dev_priv->mm.request_list);
1608         list_add_tail(&request->list, &dev_priv->mm.request_list);
1609         if (i915_file_priv) {
1610                 list_add_tail(&request->client_list,
1611                               &i915_file_priv->mm.request_list);
1612         } else {
1613                 INIT_LIST_HEAD(&request->client_list);
1614         }
1615
1616         /* Associate any objects on the flushing list matching the write
1617          * domain we're flushing with our flush.
1618          */
1619         if (flush_domains != 0) {
1620                 struct drm_i915_gem_object *obj_priv, *next;
1621
1622                 list_for_each_entry_safe(obj_priv, next,
1623                                          &dev_priv->mm.gpu_write_list,
1624                                          gpu_write_list) {
1625                         struct drm_gem_object *obj = obj_priv->obj;
1626
1627                         if ((obj->write_domain & flush_domains) ==
1628                             obj->write_domain) {
1629                                 uint32_t old_write_domain = obj->write_domain;
1630
1631                                 obj->write_domain = 0;
1632                                 list_del_init(&obj_priv->gpu_write_list);
1633                                 i915_gem_object_move_to_active(obj, seqno);
1634
1635                                 trace_i915_gem_object_change_domain(obj,
1636                                                                     obj->read_domains,
1637                                                                     old_write_domain);
1638                         }
1639                 }
1640
1641         }
1642
1643         if (!dev_priv->mm.suspended) {
1644                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1645                 if (was_empty)
1646                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1647         }
1648         return seqno;
1649 }
1650
1651 /**
1652  * Command execution barrier
1653  *
1654  * Ensures that all commands in the ring are finished
1655  * before signalling the CPU
1656  */
1657 static uint32_t
1658 i915_retire_commands(struct drm_device *dev)
1659 {
1660         drm_i915_private_t *dev_priv = dev->dev_private;
1661         uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1662         uint32_t flush_domains = 0;
1663         RING_LOCALS;
1664
1665         /* The sampler always gets flushed on i965 (sigh) */
1666         if (IS_I965G(dev))
1667                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1668         BEGIN_LP_RING(2);
1669         OUT_RING(cmd);
1670         OUT_RING(0); /* noop */
1671         ADVANCE_LP_RING();
1672         return flush_domains;
1673 }
1674
1675 /**
1676  * Moves buffers associated only with the given active seqno from the active
1677  * to inactive list, potentially freeing them.
1678  */
1679 static void
1680 i915_gem_retire_request(struct drm_device *dev,
1681                         struct drm_i915_gem_request *request)
1682 {
1683         drm_i915_private_t *dev_priv = dev->dev_private;
1684
1685         trace_i915_gem_request_retire(dev, request->seqno);
1686
1687         /* Move any buffers on the active list that are no longer referenced
1688          * by the ringbuffer to the flushing/inactive lists as appropriate.
1689          */
1690         spin_lock(&dev_priv->mm.active_list_lock);
1691         while (!list_empty(&dev_priv->mm.active_list)) {
1692                 struct drm_gem_object *obj;
1693                 struct drm_i915_gem_object *obj_priv;
1694
1695                 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1696                                             struct drm_i915_gem_object,
1697                                             list);
1698                 obj = obj_priv->obj;
1699
1700                 /* If the seqno being retired doesn't match the oldest in the
1701                  * list, then the oldest in the list must still be newer than
1702                  * this seqno.
1703                  */
1704                 if (obj_priv->last_rendering_seqno != request->seqno)
1705                         goto out;
1706
1707 #if WATCH_LRU
1708                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1709                          __func__, request->seqno, obj);
1710 #endif
1711
1712                 if (obj->write_domain != 0)
1713                         i915_gem_object_move_to_flushing(obj);
1714                 else {
1715                         /* Take a reference on the object so it won't be
1716                          * freed while the spinlock is held.  The list
1717                          * protection for this spinlock is safe when breaking
1718                          * the lock like this since the next thing we do
1719                          * is just get the head of the list again.
1720                          */
1721                         drm_gem_object_reference(obj);
1722                         i915_gem_object_move_to_inactive(obj);
1723                         spin_unlock(&dev_priv->mm.active_list_lock);
1724                         drm_gem_object_unreference(obj);
1725                         spin_lock(&dev_priv->mm.active_list_lock);
1726                 }
1727         }
1728 out:
1729         spin_unlock(&dev_priv->mm.active_list_lock);
1730 }
1731
1732 /**
1733  * Returns true if seq1 is later than seq2.
1734  */
1735 bool
1736 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1737 {
1738         return (int32_t)(seq1 - seq2) >= 0;
1739 }
1740
1741 uint32_t
1742 i915_get_gem_seqno(struct drm_device *dev)
1743 {
1744         drm_i915_private_t *dev_priv = dev->dev_private;
1745
1746         return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1747 }
1748
1749 /**
1750  * This function clears the request list as sequence numbers are passed.
1751  */
1752 void
1753 i915_gem_retire_requests(struct drm_device *dev)
1754 {
1755         drm_i915_private_t *dev_priv = dev->dev_private;
1756         uint32_t seqno;
1757
1758         if (!dev_priv->hw_status_page || list_empty(&dev_priv->mm.request_list))
1759                 return;
1760
1761         seqno = i915_get_gem_seqno(dev);
1762
1763         while (!list_empty(&dev_priv->mm.request_list)) {
1764                 struct drm_i915_gem_request *request;
1765                 uint32_t retiring_seqno;
1766
1767                 request = list_first_entry(&dev_priv->mm.request_list,
1768                                            struct drm_i915_gem_request,
1769                                            list);
1770                 retiring_seqno = request->seqno;
1771
1772                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1773                     atomic_read(&dev_priv->mm.wedged)) {
1774                         i915_gem_retire_request(dev, request);
1775
1776                         list_del(&request->list);
1777                         list_del(&request->client_list);
1778                         kfree(request);
1779                 } else
1780                         break;
1781         }
1782
1783         if (unlikely (dev_priv->trace_irq_seqno &&
1784                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1785                 i915_user_irq_put(dev);
1786                 dev_priv->trace_irq_seqno = 0;
1787         }
1788 }
1789
1790 void
1791 i915_gem_retire_work_handler(struct work_struct *work)
1792 {
1793         drm_i915_private_t *dev_priv;
1794         struct drm_device *dev;
1795
1796         dev_priv = container_of(work, drm_i915_private_t,
1797                                 mm.retire_work.work);
1798         dev = dev_priv->dev;
1799
1800         mutex_lock(&dev->struct_mutex);
1801         i915_gem_retire_requests(dev);
1802         if (!dev_priv->mm.suspended &&
1803             !list_empty(&dev_priv->mm.request_list))
1804                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1805         mutex_unlock(&dev->struct_mutex);
1806 }
1807
1808 int
1809 i915_do_wait_request(struct drm_device *dev, uint32_t seqno, int interruptible)
1810 {
1811         drm_i915_private_t *dev_priv = dev->dev_private;
1812         u32 ier;
1813         int ret = 0;
1814
1815         BUG_ON(seqno == 0);
1816
1817         if (atomic_read(&dev_priv->mm.wedged))
1818                 return -EIO;
1819
1820         if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
1821                 if (HAS_PCH_SPLIT(dev))
1822                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1823                 else
1824                         ier = I915_READ(IER);
1825                 if (!ier) {
1826                         DRM_ERROR("something (likely vbetool) disabled "
1827                                   "interrupts, re-enabling\n");
1828                         i915_driver_irq_preinstall(dev);
1829                         i915_driver_irq_postinstall(dev);
1830                 }
1831
1832                 trace_i915_gem_request_wait_begin(dev, seqno);
1833
1834                 dev_priv->mm.waiting_gem_seqno = seqno;
1835                 i915_user_irq_get(dev);
1836                 if (interruptible)
1837                         ret = wait_event_interruptible(dev_priv->irq_queue,
1838                                 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1839                                 atomic_read(&dev_priv->mm.wedged));
1840                 else
1841                         wait_event(dev_priv->irq_queue,
1842                                 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1843                                 atomic_read(&dev_priv->mm.wedged));
1844
1845                 i915_user_irq_put(dev);
1846                 dev_priv->mm.waiting_gem_seqno = 0;
1847
1848                 trace_i915_gem_request_wait_end(dev, seqno);
1849         }
1850         if (atomic_read(&dev_priv->mm.wedged))
1851                 ret = -EIO;
1852
1853         if (ret && ret != -ERESTARTSYS)
1854                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1855                           __func__, ret, seqno, i915_get_gem_seqno(dev));
1856
1857         /* Directly dispatch request retiring.  While we have the work queue
1858          * to handle this, the waiter on a request often wants an associated
1859          * buffer to have made it to the inactive list, and we would need
1860          * a separate wait queue to handle that.
1861          */
1862         if (ret == 0)
1863                 i915_gem_retire_requests(dev);
1864
1865         return ret;
1866 }
1867
1868 /**
1869  * Waits for a sequence number to be signaled, and cleans up the
1870  * request and object lists appropriately for that event.
1871  */
1872 static int
1873 i915_wait_request(struct drm_device *dev, uint32_t seqno)
1874 {
1875         return i915_do_wait_request(dev, seqno, 1);
1876 }
1877
1878 static void
1879 i915_gem_flush(struct drm_device *dev,
1880                uint32_t invalidate_domains,
1881                uint32_t flush_domains)
1882 {
1883         drm_i915_private_t *dev_priv = dev->dev_private;
1884         uint32_t cmd;
1885         RING_LOCALS;
1886
1887 #if WATCH_EXEC
1888         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1889                   invalidate_domains, flush_domains);
1890 #endif
1891         trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
1892                                      invalidate_domains, flush_domains);
1893
1894         if (flush_domains & I915_GEM_DOMAIN_CPU)
1895                 drm_agp_chipset_flush(dev);
1896
1897         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
1898                 /*
1899                  * read/write caches:
1900                  *
1901                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1902                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
1903                  * also flushed at 2d versus 3d pipeline switches.
1904                  *
1905                  * read-only caches:
1906                  *
1907                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1908                  * MI_READ_FLUSH is set, and is always flushed on 965.
1909                  *
1910                  * I915_GEM_DOMAIN_COMMAND may not exist?
1911                  *
1912                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1913                  * invalidated when MI_EXE_FLUSH is set.
1914                  *
1915                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1916                  * invalidated with every MI_FLUSH.
1917                  *
1918                  * TLBs:
1919                  *
1920                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1921                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1922                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1923                  * are flushed at any MI_FLUSH.
1924                  */
1925
1926                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1927                 if ((invalidate_domains|flush_domains) &
1928                     I915_GEM_DOMAIN_RENDER)
1929                         cmd &= ~MI_NO_WRITE_FLUSH;
1930                 if (!IS_I965G(dev)) {
1931                         /*
1932                          * On the 965, the sampler cache always gets flushed
1933                          * and this bit is reserved.
1934                          */
1935                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1936                                 cmd |= MI_READ_FLUSH;
1937                 }
1938                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1939                         cmd |= MI_EXE_FLUSH;
1940
1941 #if WATCH_EXEC
1942                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1943 #endif
1944                 BEGIN_LP_RING(2);
1945                 OUT_RING(cmd);
1946                 OUT_RING(MI_NOOP);
1947                 ADVANCE_LP_RING();
1948         }
1949 }
1950
1951 /**
1952  * Ensures that all rendering to the object has completed and the object is
1953  * safe to unbind from the GTT or access from the CPU.
1954  */
1955 static int
1956 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1957 {
1958         struct drm_device *dev = obj->dev;
1959         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1960         int ret;
1961
1962         /* This function only exists to support waiting for existing rendering,
1963          * not for emitting required flushes.
1964          */
1965         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1966
1967         /* If there is rendering queued on the buffer being evicted, wait for
1968          * it.
1969          */
1970         if (obj_priv->active) {
1971 #if WATCH_BUF
1972                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1973                           __func__, obj, obj_priv->last_rendering_seqno);
1974 #endif
1975                 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1976                 if (ret != 0)
1977                         return ret;
1978         }
1979
1980         return 0;
1981 }
1982
1983 /**
1984  * Unbinds an object from the GTT aperture.
1985  */
1986 int
1987 i915_gem_object_unbind(struct drm_gem_object *obj)
1988 {
1989         struct drm_device *dev = obj->dev;
1990         drm_i915_private_t *dev_priv = dev->dev_private;
1991         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1992         int ret = 0;
1993
1994 #if WATCH_BUF
1995         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1996         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1997 #endif
1998         if (obj_priv->gtt_space == NULL)
1999                 return 0;
2000
2001         if (obj_priv->pin_count != 0) {
2002                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2003                 return -EINVAL;
2004         }
2005
2006         /* blow away mappings if mapped through GTT */
2007         i915_gem_release_mmap(obj);
2008
2009         /* Move the object to the CPU domain to ensure that
2010          * any possible CPU writes while it's not in the GTT
2011          * are flushed when we go to remap it. This will
2012          * also ensure that all pending GPU writes are finished
2013          * before we unbind.
2014          */
2015         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2016         if (ret) {
2017                 if (ret != -ERESTARTSYS)
2018                         DRM_ERROR("set_domain failed: %d\n", ret);
2019                 return ret;
2020         }
2021
2022         BUG_ON(obj_priv->active);
2023
2024         /* release the fence reg _after_ flushing */
2025         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2026                 i915_gem_clear_fence_reg(obj);
2027
2028         if (obj_priv->agp_mem != NULL) {
2029                 drm_unbind_agp(obj_priv->agp_mem);
2030                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2031                 obj_priv->agp_mem = NULL;
2032         }
2033
2034         i915_gem_object_put_pages(obj);
2035         BUG_ON(obj_priv->pages_refcount);
2036
2037         if (obj_priv->gtt_space) {
2038                 atomic_dec(&dev->gtt_count);
2039                 atomic_sub(obj->size, &dev->gtt_memory);
2040
2041                 drm_mm_put_block(obj_priv->gtt_space);
2042                 obj_priv->gtt_space = NULL;
2043         }
2044
2045         /* Remove ourselves from the LRU list if present. */
2046         spin_lock(&dev_priv->mm.active_list_lock);
2047         if (!list_empty(&obj_priv->list))
2048                 list_del_init(&obj_priv->list);
2049         spin_unlock(&dev_priv->mm.active_list_lock);
2050
2051         if (i915_gem_object_is_purgeable(obj_priv))
2052                 i915_gem_object_truncate(obj);
2053
2054         trace_i915_gem_object_unbind(obj);
2055
2056         return 0;
2057 }
2058
2059 static struct drm_gem_object *
2060 i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2061 {
2062         drm_i915_private_t *dev_priv = dev->dev_private;
2063         struct drm_i915_gem_object *obj_priv;
2064         struct drm_gem_object *best = NULL;
2065         struct drm_gem_object *first = NULL;
2066
2067         /* Try to find the smallest clean object */
2068         list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
2069                 struct drm_gem_object *obj = obj_priv->obj;
2070                 if (obj->size >= min_size) {
2071                         if ((!obj_priv->dirty ||
2072                              i915_gem_object_is_purgeable(obj_priv)) &&
2073                             (!best || obj->size < best->size)) {
2074                                 best = obj;
2075                                 if (best->size == min_size)
2076                                         return best;
2077                         }
2078                         if (!first)
2079                             first = obj;
2080                 }
2081         }
2082
2083         return best ? best : first;
2084 }
2085
2086 static int
2087 i915_gem_evict_everything(struct drm_device *dev)
2088 {
2089         drm_i915_private_t *dev_priv = dev->dev_private;
2090         int ret;
2091         uint32_t seqno;
2092         bool lists_empty;
2093
2094         spin_lock(&dev_priv->mm.active_list_lock);
2095         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2096                        list_empty(&dev_priv->mm.flushing_list) &&
2097                        list_empty(&dev_priv->mm.active_list));
2098         spin_unlock(&dev_priv->mm.active_list_lock);
2099
2100         if (lists_empty)
2101                 return -ENOSPC;
2102
2103         /* Flush everything (on to the inactive lists) and evict */
2104         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2105         seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
2106         if (seqno == 0)
2107                 return -ENOMEM;
2108
2109         ret = i915_wait_request(dev, seqno);
2110         if (ret)
2111                 return ret;
2112
2113         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2114
2115         ret = i915_gem_evict_from_inactive_list(dev);
2116         if (ret)
2117                 return ret;
2118
2119         spin_lock(&dev_priv->mm.active_list_lock);
2120         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2121                        list_empty(&dev_priv->mm.flushing_list) &&
2122                        list_empty(&dev_priv->mm.active_list));
2123         spin_unlock(&dev_priv->mm.active_list_lock);
2124         BUG_ON(!lists_empty);
2125
2126         return 0;
2127 }
2128
2129 static int
2130 i915_gem_evict_something(struct drm_device *dev, int min_size)
2131 {
2132         drm_i915_private_t *dev_priv = dev->dev_private;
2133         struct drm_gem_object *obj;
2134         int ret;
2135
2136         for (;;) {
2137                 i915_gem_retire_requests(dev);
2138
2139                 /* If there's an inactive buffer available now, grab it
2140                  * and be done.
2141                  */
2142                 obj = i915_gem_find_inactive_object(dev, min_size);
2143                 if (obj) {
2144                         struct drm_i915_gem_object *obj_priv;
2145
2146 #if WATCH_LRU
2147                         DRM_INFO("%s: evicting %p\n", __func__, obj);
2148 #endif
2149                         obj_priv = obj->driver_private;
2150                         BUG_ON(obj_priv->pin_count != 0);
2151                         BUG_ON(obj_priv->active);
2152
2153                         /* Wait on the rendering and unbind the buffer. */
2154                         return i915_gem_object_unbind(obj);
2155                 }
2156
2157                 /* If we didn't get anything, but the ring is still processing
2158                  * things, wait for the next to finish and hopefully leave us
2159                  * a buffer to evict.
2160                  */
2161                 if (!list_empty(&dev_priv->mm.request_list)) {
2162                         struct drm_i915_gem_request *request;
2163
2164                         request = list_first_entry(&dev_priv->mm.request_list,
2165                                                    struct drm_i915_gem_request,
2166                                                    list);
2167
2168                         ret = i915_wait_request(dev, request->seqno);
2169                         if (ret)
2170                                 return ret;
2171
2172                         continue;
2173                 }
2174
2175                 /* If we didn't have anything on the request list but there
2176                  * are buffers awaiting a flush, emit one and try again.
2177                  * When we wait on it, those buffers waiting for that flush
2178                  * will get moved to inactive.
2179                  */
2180                 if (!list_empty(&dev_priv->mm.flushing_list)) {
2181                         struct drm_i915_gem_object *obj_priv;
2182
2183                         /* Find an object that we can immediately reuse */
2184                         list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
2185                                 obj = obj_priv->obj;
2186                                 if (obj->size >= min_size)
2187                                         break;
2188
2189                                 obj = NULL;
2190                         }
2191
2192                         if (obj != NULL) {
2193                                 uint32_t seqno;
2194
2195                                 i915_gem_flush(dev,
2196                                                obj->write_domain,
2197                                                obj->write_domain);
2198                                 seqno = i915_add_request(dev, NULL, obj->write_domain);
2199                                 if (seqno == 0)
2200                                         return -ENOMEM;
2201
2202                                 ret = i915_wait_request(dev, seqno);
2203                                 if (ret)
2204                                         return ret;
2205
2206                                 continue;
2207                         }
2208                 }
2209
2210                 /* If we didn't do any of the above, there's no single buffer
2211                  * large enough to swap out for the new one, so just evict
2212                  * everything and start again. (This should be rare.)
2213                  */
2214                 if (!list_empty (&dev_priv->mm.inactive_list))
2215                         return i915_gem_evict_from_inactive_list(dev);
2216                 else
2217                         return i915_gem_evict_everything(dev);
2218         }
2219 }
2220
2221 int
2222 i915_gem_object_get_pages(struct drm_gem_object *obj,
2223                           gfp_t gfpmask)
2224 {
2225         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2226         int page_count, i;
2227         struct address_space *mapping;
2228         struct inode *inode;
2229         struct page *page;
2230         int ret;
2231
2232         if (obj_priv->pages_refcount++ != 0)
2233                 return 0;
2234
2235         /* Get the list of pages out of our struct file.  They'll be pinned
2236          * at this point until we release them.
2237          */
2238         page_count = obj->size / PAGE_SIZE;
2239         BUG_ON(obj_priv->pages != NULL);
2240         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2241         if (obj_priv->pages == NULL) {
2242                 obj_priv->pages_refcount--;
2243                 return -ENOMEM;
2244         }
2245
2246         inode = obj->filp->f_path.dentry->d_inode;
2247         mapping = inode->i_mapping;
2248         for (i = 0; i < page_count; i++) {
2249                 page = read_cache_page_gfp(mapping, i,
2250                                            mapping_gfp_mask (mapping) |
2251                                            __GFP_COLD |
2252                                            gfpmask);
2253                 if (IS_ERR(page)) {
2254                         ret = PTR_ERR(page);
2255                         i915_gem_object_put_pages(obj);
2256                         return ret;
2257                 }
2258                 obj_priv->pages[i] = page;
2259         }
2260
2261         if (obj_priv->tiling_mode != I915_TILING_NONE)
2262                 i915_gem_object_do_bit_17_swizzle(obj);
2263
2264         return 0;
2265 }
2266
2267 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2268 {
2269         struct drm_gem_object *obj = reg->obj;
2270         struct drm_device *dev = obj->dev;
2271         drm_i915_private_t *dev_priv = dev->dev_private;
2272         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2273         int regnum = obj_priv->fence_reg;
2274         uint64_t val;
2275
2276         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2277                     0xfffff000) << 32;
2278         val |= obj_priv->gtt_offset & 0xfffff000;
2279         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2280                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2281
2282         if (obj_priv->tiling_mode == I915_TILING_Y)
2283                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2284         val |= I965_FENCE_REG_VALID;
2285
2286         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2287 }
2288
2289 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2290 {
2291         struct drm_gem_object *obj = reg->obj;
2292         struct drm_device *dev = obj->dev;
2293         drm_i915_private_t *dev_priv = dev->dev_private;
2294         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2295         int regnum = obj_priv->fence_reg;
2296         uint64_t val;
2297
2298         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2299                     0xfffff000) << 32;
2300         val |= obj_priv->gtt_offset & 0xfffff000;
2301         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2302         if (obj_priv->tiling_mode == I915_TILING_Y)
2303                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2304         val |= I965_FENCE_REG_VALID;
2305
2306         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2307 }
2308
2309 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2310 {
2311         struct drm_gem_object *obj = reg->obj;
2312         struct drm_device *dev = obj->dev;
2313         drm_i915_private_t *dev_priv = dev->dev_private;
2314         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2315         int regnum = obj_priv->fence_reg;
2316         int tile_width;
2317         uint32_t fence_reg, val;
2318         uint32_t pitch_val;
2319
2320         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2321             (obj_priv->gtt_offset & (obj->size - 1))) {
2322                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2323                      __func__, obj_priv->gtt_offset, obj->size);
2324                 return;
2325         }
2326
2327         if (obj_priv->tiling_mode == I915_TILING_Y &&
2328             HAS_128_BYTE_Y_TILING(dev))
2329                 tile_width = 128;
2330         else
2331                 tile_width = 512;
2332
2333         /* Note: pitch better be a power of two tile widths */
2334         pitch_val = obj_priv->stride / tile_width;
2335         pitch_val = ffs(pitch_val) - 1;
2336
2337         val = obj_priv->gtt_offset;
2338         if (obj_priv->tiling_mode == I915_TILING_Y)
2339                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2340         val |= I915_FENCE_SIZE_BITS(obj->size);
2341         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2342         val |= I830_FENCE_REG_VALID;
2343
2344         if (regnum < 8)
2345                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2346         else
2347                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2348         I915_WRITE(fence_reg, val);
2349 }
2350
2351 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2352 {
2353         struct drm_gem_object *obj = reg->obj;
2354         struct drm_device *dev = obj->dev;
2355         drm_i915_private_t *dev_priv = dev->dev_private;
2356         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2357         int regnum = obj_priv->fence_reg;
2358         uint32_t val;
2359         uint32_t pitch_val;
2360         uint32_t fence_size_bits;
2361
2362         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2363             (obj_priv->gtt_offset & (obj->size - 1))) {
2364                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2365                      __func__, obj_priv->gtt_offset);
2366                 return;
2367         }
2368
2369         pitch_val = obj_priv->stride / 128;
2370         pitch_val = ffs(pitch_val) - 1;
2371         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2372
2373         val = obj_priv->gtt_offset;
2374         if (obj_priv->tiling_mode == I915_TILING_Y)
2375                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2376         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2377         WARN_ON(fence_size_bits & ~0x00000f00);
2378         val |= fence_size_bits;
2379         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2380         val |= I830_FENCE_REG_VALID;
2381
2382         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2383 }
2384
2385 /**
2386  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2387  * @obj: object to map through a fence reg
2388  *
2389  * When mapping objects through the GTT, userspace wants to be able to write
2390  * to them without having to worry about swizzling if the object is tiled.
2391  *
2392  * This function walks the fence regs looking for a free one for @obj,
2393  * stealing one if it can't find any.
2394  *
2395  * It then sets up the reg based on the object's properties: address, pitch
2396  * and tiling format.
2397  */
2398 int
2399 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2400 {
2401         struct drm_device *dev = obj->dev;
2402         struct drm_i915_private *dev_priv = dev->dev_private;
2403         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2404         struct drm_i915_fence_reg *reg = NULL;
2405         struct drm_i915_gem_object *old_obj_priv = NULL;
2406         int i, ret, avail;
2407
2408         /* Just update our place in the LRU if our fence is getting used. */
2409         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2410                 list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2411                 return 0;
2412         }
2413
2414         switch (obj_priv->tiling_mode) {
2415         case I915_TILING_NONE:
2416                 WARN(1, "allocating a fence for non-tiled object?\n");
2417                 break;
2418         case I915_TILING_X:
2419                 if (!obj_priv->stride)
2420                         return -EINVAL;
2421                 WARN((obj_priv->stride & (512 - 1)),
2422                      "object 0x%08x is X tiled but has non-512B pitch\n",
2423                      obj_priv->gtt_offset);
2424                 break;
2425         case I915_TILING_Y:
2426                 if (!obj_priv->stride)
2427                         return -EINVAL;
2428                 WARN((obj_priv->stride & (128 - 1)),
2429                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2430                      obj_priv->gtt_offset);
2431                 break;
2432         }
2433
2434         /* First try to find a free reg */
2435         avail = 0;
2436         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2437                 reg = &dev_priv->fence_regs[i];
2438                 if (!reg->obj)
2439                         break;
2440
2441                 old_obj_priv = reg->obj->driver_private;
2442                 if (!old_obj_priv->pin_count)
2443                     avail++;
2444         }
2445
2446         /* None available, try to steal one or wait for a user to finish */
2447         if (i == dev_priv->num_fence_regs) {
2448                 struct drm_gem_object *old_obj = NULL;
2449
2450                 if (avail == 0)
2451                         return -ENOSPC;
2452
2453                 list_for_each_entry(old_obj_priv, &dev_priv->mm.fence_list,
2454                                     fence_list) {
2455                         old_obj = old_obj_priv->obj;
2456
2457                         if (old_obj_priv->pin_count)
2458                                 continue;
2459
2460                         /* Take a reference, as otherwise the wait_rendering
2461                          * below may cause the object to get freed out from
2462                          * under us.
2463                          */
2464                         drm_gem_object_reference(old_obj);
2465
2466                         break;
2467                 }
2468
2469                 i = old_obj_priv->fence_reg;
2470                 reg = &dev_priv->fence_regs[i];
2471
2472                 ret = i915_gem_object_put_fence_reg(old_obj);
2473                 drm_gem_object_unreference(old_obj);
2474                 if (ret != 0)
2475                         return ret;
2476         }
2477
2478         obj_priv->fence_reg = i;
2479         list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2480
2481         reg->obj = obj;
2482
2483         if (IS_GEN6(dev))
2484                 sandybridge_write_fence_reg(reg);
2485         else if (IS_I965G(dev))
2486                 i965_write_fence_reg(reg);
2487         else if (IS_I9XX(dev))
2488                 i915_write_fence_reg(reg);
2489         else
2490                 i830_write_fence_reg(reg);
2491
2492         trace_i915_gem_object_get_fence(obj, i, obj_priv->tiling_mode);
2493
2494         return 0;
2495 }
2496
2497 /**
2498  * i915_gem_clear_fence_reg - clear out fence register info
2499  * @obj: object to clear
2500  *
2501  * Zeroes out the fence register itself and clears out the associated
2502  * data structures in dev_priv and obj_priv.
2503  */
2504 static void
2505 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2506 {
2507         struct drm_device *dev = obj->dev;
2508         drm_i915_private_t *dev_priv = dev->dev_private;
2509         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2510
2511         if (IS_GEN6(dev)) {
2512                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2513                              (obj_priv->fence_reg * 8), 0);
2514         } else if (IS_I965G(dev)) {
2515                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2516         } else {
2517                 uint32_t fence_reg;
2518
2519                 if (obj_priv->fence_reg < 8)
2520                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2521                 else
2522                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2523                                                        8) * 4;
2524
2525                 I915_WRITE(fence_reg, 0);
2526         }
2527
2528         dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2529         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2530         list_del_init(&obj_priv->fence_list);
2531 }
2532
2533 /**
2534  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2535  * to the buffer to finish, and then resets the fence register.
2536  * @obj: tiled object holding a fence register.
2537  *
2538  * Zeroes out the fence register itself and clears out the associated
2539  * data structures in dev_priv and obj_priv.
2540  */
2541 int
2542 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2543 {
2544         struct drm_device *dev = obj->dev;
2545         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2546
2547         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2548                 return 0;
2549
2550         /* If we've changed tiling, GTT-mappings of the object
2551          * need to re-fault to ensure that the correct fence register
2552          * setup is in place.
2553          */
2554         i915_gem_release_mmap(obj);
2555
2556         /* On the i915, GPU access to tiled buffers is via a fence,
2557          * therefore we must wait for any outstanding access to complete
2558          * before clearing the fence.
2559          */
2560         if (!IS_I965G(dev)) {
2561                 int ret;
2562
2563                 i915_gem_object_flush_gpu_write_domain(obj);
2564                 ret = i915_gem_object_wait_rendering(obj);
2565                 if (ret != 0)
2566                         return ret;
2567         }
2568
2569         i915_gem_object_flush_gtt_write_domain(obj);
2570         i915_gem_clear_fence_reg (obj);
2571
2572         return 0;
2573 }
2574
2575 /**
2576  * Finds free space in the GTT aperture and binds the object there.
2577  */
2578 static int
2579 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2580 {
2581         struct drm_device *dev = obj->dev;
2582         drm_i915_private_t *dev_priv = dev->dev_private;
2583         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2584         struct drm_mm_node *free_space;
2585         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2586         int ret;
2587
2588         if (obj_priv->madv != I915_MADV_WILLNEED) {
2589                 DRM_ERROR("Attempting to bind a purgeable object\n");
2590                 return -EINVAL;
2591         }
2592
2593         if (alignment == 0)
2594                 alignment = i915_gem_get_gtt_alignment(obj);
2595         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2596                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2597                 return -EINVAL;
2598         }
2599
2600  search_free:
2601         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2602                                         obj->size, alignment, 0);
2603         if (free_space != NULL) {
2604                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2605                                                        alignment);
2606                 if (obj_priv->gtt_space != NULL) {
2607                         obj_priv->gtt_space->private = obj;
2608                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2609                 }
2610         }
2611         if (obj_priv->gtt_space == NULL) {
2612                 /* If the gtt is empty and we're still having trouble
2613                  * fitting our object in, we're out of memory.
2614                  */
2615 #if WATCH_LRU
2616                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2617 #endif
2618                 ret = i915_gem_evict_something(dev, obj->size);
2619                 if (ret)
2620                         return ret;
2621
2622                 goto search_free;
2623         }
2624
2625 #if WATCH_BUF
2626         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2627                  obj->size, obj_priv->gtt_offset);
2628 #endif
2629         ret = i915_gem_object_get_pages(obj, gfpmask);
2630         if (ret) {
2631                 drm_mm_put_block(obj_priv->gtt_space);
2632                 obj_priv->gtt_space = NULL;
2633
2634                 if (ret == -ENOMEM) {
2635                         /* first try to clear up some space from the GTT */
2636                         ret = i915_gem_evict_something(dev, obj->size);
2637                         if (ret) {
2638                                 /* now try to shrink everyone else */
2639                                 if (gfpmask) {
2640                                         gfpmask = 0;
2641                                         goto search_free;
2642                                 }
2643
2644                                 return ret;
2645                         }
2646
2647                         goto search_free;
2648                 }
2649
2650                 return ret;
2651         }
2652
2653         /* Create an AGP memory structure pointing at our pages, and bind it
2654          * into the GTT.
2655          */
2656         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2657                                                obj_priv->pages,
2658                                                obj->size >> PAGE_SHIFT,
2659                                                obj_priv->gtt_offset,
2660                                                obj_priv->agp_type);
2661         if (obj_priv->agp_mem == NULL) {
2662                 i915_gem_object_put_pages(obj);
2663                 drm_mm_put_block(obj_priv->gtt_space);
2664                 obj_priv->gtt_space = NULL;
2665
2666                 ret = i915_gem_evict_something(dev, obj->size);
2667                 if (ret)
2668                         return ret;
2669
2670                 goto search_free;
2671         }
2672         atomic_inc(&dev->gtt_count);
2673         atomic_add(obj->size, &dev->gtt_memory);
2674
2675         /* Assert that the object is not currently in any GPU domain. As it
2676          * wasn't in the GTT, there shouldn't be any way it could have been in
2677          * a GPU cache
2678          */
2679         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2680         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2681
2682         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2683
2684         return 0;
2685 }
2686
2687 void
2688 i915_gem_clflush_object(struct drm_gem_object *obj)
2689 {
2690         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
2691
2692         /* If we don't have a page list set up, then we're not pinned
2693          * to GPU, and we can ignore the cache flush because it'll happen
2694          * again at bind time.
2695          */
2696         if (obj_priv->pages == NULL)
2697                 return;
2698
2699         trace_i915_gem_object_clflush(obj);
2700
2701         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2702 }
2703
2704 /** Flushes any GPU write domain for the object if it's dirty. */
2705 static void
2706 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2707 {
2708         struct drm_device *dev = obj->dev;
2709         uint32_t seqno;
2710         uint32_t old_write_domain;
2711
2712         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2713                 return;
2714
2715         /* Queue the GPU write cache flushing we need. */
2716         old_write_domain = obj->write_domain;
2717         i915_gem_flush(dev, 0, obj->write_domain);
2718         seqno = i915_add_request(dev, NULL, obj->write_domain);
2719         BUG_ON(obj->write_domain);
2720         i915_gem_object_move_to_active(obj, seqno);
2721
2722         trace_i915_gem_object_change_domain(obj,
2723                                             obj->read_domains,
2724                                             old_write_domain);
2725 }
2726
2727 /** Flushes the GTT write domain for the object if it's dirty. */
2728 static void
2729 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2730 {
2731         uint32_t old_write_domain;
2732
2733         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2734                 return;
2735
2736         /* No actual flushing is required for the GTT write domain.   Writes
2737          * to it immediately go to main memory as far as we know, so there's
2738          * no chipset flush.  It also doesn't land in render cache.
2739          */
2740         old_write_domain = obj->write_domain;
2741         obj->write_domain = 0;
2742
2743         trace_i915_gem_object_change_domain(obj,
2744                                             obj->read_domains,
2745                                             old_write_domain);
2746 }
2747
2748 /** Flushes the CPU write domain for the object if it's dirty. */
2749 static void
2750 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2751 {
2752         struct drm_device *dev = obj->dev;
2753         uint32_t old_write_domain;
2754
2755         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2756                 return;
2757
2758         i915_gem_clflush_object(obj);
2759         drm_agp_chipset_flush(dev);
2760         old_write_domain = obj->write_domain;
2761         obj->write_domain = 0;
2762
2763         trace_i915_gem_object_change_domain(obj,
2764                                             obj->read_domains,
2765                                             old_write_domain);
2766 }
2767
2768 void
2769 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2770 {
2771         switch (obj->write_domain) {
2772         case I915_GEM_DOMAIN_GTT:
2773                 i915_gem_object_flush_gtt_write_domain(obj);
2774                 break;
2775         case I915_GEM_DOMAIN_CPU:
2776                 i915_gem_object_flush_cpu_write_domain(obj);
2777                 break;
2778         default:
2779                 i915_gem_object_flush_gpu_write_domain(obj);
2780                 break;
2781         }
2782 }
2783
2784 /**
2785  * Moves a single object to the GTT read, and possibly write domain.
2786  *
2787  * This function returns when the move is complete, including waiting on
2788  * flushes to occur.
2789  */
2790 int
2791 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2792 {
2793         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2794         uint32_t old_write_domain, old_read_domains;
2795         int ret;
2796
2797         /* Not valid to be called on unbound objects. */
2798         if (obj_priv->gtt_space == NULL)
2799                 return -EINVAL;
2800
2801         i915_gem_object_flush_gpu_write_domain(obj);
2802         /* Wait on any GPU rendering and flushing to occur. */
2803         ret = i915_gem_object_wait_rendering(obj);
2804         if (ret != 0)
2805                 return ret;
2806
2807         old_write_domain = obj->write_domain;
2808         old_read_domains = obj->read_domains;
2809
2810         /* If we're writing through the GTT domain, then CPU and GPU caches
2811          * will need to be invalidated at next use.
2812          */
2813         if (write)
2814                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2815
2816         i915_gem_object_flush_cpu_write_domain(obj);
2817
2818         /* It should now be out of any other write domains, and we can update
2819          * the domain values for our changes.
2820          */
2821         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2822         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2823         if (write) {
2824                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2825                 obj_priv->dirty = 1;
2826         }
2827
2828         trace_i915_gem_object_change_domain(obj,
2829                                             old_read_domains,
2830                                             old_write_domain);
2831
2832         return 0;
2833 }
2834
2835 /*
2836  * Prepare buffer for display plane. Use uninterruptible for possible flush
2837  * wait, as in modesetting process we're not supposed to be interrupted.
2838  */
2839 int
2840 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2841 {
2842         struct drm_device *dev = obj->dev;
2843         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2844         uint32_t old_write_domain, old_read_domains;
2845         int ret;
2846
2847         /* Not valid to be called on unbound objects. */
2848         if (obj_priv->gtt_space == NULL)
2849                 return -EINVAL;
2850
2851         i915_gem_object_flush_gpu_write_domain(obj);
2852
2853         /* Wait on any GPU rendering and flushing to occur. */
2854         if (obj_priv->active) {
2855 #if WATCH_BUF
2856                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2857                           __func__, obj, obj_priv->last_rendering_seqno);
2858 #endif
2859                 ret = i915_do_wait_request(dev, obj_priv->last_rendering_seqno, 0);
2860                 if (ret != 0)
2861                         return ret;
2862         }
2863
2864         old_write_domain = obj->write_domain;
2865         old_read_domains = obj->read_domains;
2866
2867         obj->read_domains &= I915_GEM_DOMAIN_GTT;
2868
2869         i915_gem_object_flush_cpu_write_domain(obj);
2870
2871         /* It should now be out of any other write domains, and we can update
2872          * the domain values for our changes.
2873          */
2874         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2875         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2876         obj->write_domain = I915_GEM_DOMAIN_GTT;
2877         obj_priv->dirty = 1;
2878
2879         trace_i915_gem_object_change_domain(obj,
2880                                             old_read_domains,
2881                                             old_write_domain);
2882
2883         return 0;
2884 }
2885
2886 /**
2887  * Moves a single object to the CPU read, and possibly write domain.
2888  *
2889  * This function returns when the move is complete, including waiting on
2890  * flushes to occur.
2891  */
2892 static int
2893 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2894 {
2895         uint32_t old_write_domain, old_read_domains;
2896         int ret;
2897
2898         i915_gem_object_flush_gpu_write_domain(obj);
2899         /* Wait on any GPU rendering and flushing to occur. */
2900         ret = i915_gem_object_wait_rendering(obj);
2901         if (ret != 0)
2902                 return ret;
2903
2904         i915_gem_object_flush_gtt_write_domain(obj);
2905
2906         /* If we have a partially-valid cache of the object in the CPU,
2907          * finish invalidating it and free the per-page flags.
2908          */
2909         i915_gem_object_set_to_full_cpu_read_domain(obj);
2910
2911         old_write_domain = obj->write_domain;
2912         old_read_domains = obj->read_domains;
2913
2914         /* Flush the CPU cache if it's still invalid. */
2915         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2916                 i915_gem_clflush_object(obj);
2917
2918                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2919         }
2920
2921         /* It should now be out of any other write domains, and we can update
2922          * the domain values for our changes.
2923          */
2924         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2925
2926         /* If we're writing through the CPU, then the GPU read domains will
2927          * need to be invalidated at next use.
2928          */
2929         if (write) {
2930                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2931                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2932         }
2933
2934         trace_i915_gem_object_change_domain(obj,
2935                                             old_read_domains,
2936                                             old_write_domain);
2937
2938         return 0;
2939 }
2940
2941 /*
2942  * Set the next domain for the specified object. This
2943  * may not actually perform the necessary flushing/invaliding though,
2944  * as that may want to be batched with other set_domain operations
2945  *
2946  * This is (we hope) the only really tricky part of gem. The goal
2947  * is fairly simple -- track which caches hold bits of the object
2948  * and make sure they remain coherent. A few concrete examples may
2949  * help to explain how it works. For shorthand, we use the notation
2950  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2951  * a pair of read and write domain masks.
2952  *
2953  * Case 1: the batch buffer
2954  *
2955  *      1. Allocated
2956  *      2. Written by CPU
2957  *      3. Mapped to GTT
2958  *      4. Read by GPU
2959  *      5. Unmapped from GTT
2960  *      6. Freed
2961  *
2962  *      Let's take these a step at a time
2963  *
2964  *      1. Allocated
2965  *              Pages allocated from the kernel may still have
2966  *              cache contents, so we set them to (CPU, CPU) always.
2967  *      2. Written by CPU (using pwrite)
2968  *              The pwrite function calls set_domain (CPU, CPU) and
2969  *              this function does nothing (as nothing changes)
2970  *      3. Mapped by GTT
2971  *              This function asserts that the object is not
2972  *              currently in any GPU-based read or write domains
2973  *      4. Read by GPU
2974  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2975  *              As write_domain is zero, this function adds in the
2976  *              current read domains (CPU+COMMAND, 0).
2977  *              flush_domains is set to CPU.
2978  *              invalidate_domains is set to COMMAND
2979  *              clflush is run to get data out of the CPU caches
2980  *              then i915_dev_set_domain calls i915_gem_flush to
2981  *              emit an MI_FLUSH and drm_agp_chipset_flush
2982  *      5. Unmapped from GTT
2983  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2984  *              flush_domains and invalidate_domains end up both zero
2985  *              so no flushing/invalidating happens
2986  *      6. Freed
2987  *              yay, done
2988  *
2989  * Case 2: The shared render buffer
2990  *
2991  *      1. Allocated
2992  *      2. Mapped to GTT
2993  *      3. Read/written by GPU
2994  *      4. set_domain to (CPU,CPU)
2995  *      5. Read/written by CPU
2996  *      6. Read/written by GPU
2997  *
2998  *      1. Allocated
2999  *              Same as last example, (CPU, CPU)
3000  *      2. Mapped to GTT
3001  *              Nothing changes (assertions find that it is not in the GPU)
3002  *      3. Read/written by GPU
3003  *              execbuffer calls set_domain (RENDER, RENDER)
3004  *              flush_domains gets CPU
3005  *              invalidate_domains gets GPU
3006  *              clflush (obj)
3007  *              MI_FLUSH and drm_agp_chipset_flush
3008  *      4. set_domain (CPU, CPU)
3009  *              flush_domains gets GPU
3010  *              invalidate_domains gets CPU
3011  *              wait_rendering (obj) to make sure all drawing is complete.
3012  *              This will include an MI_FLUSH to get the data from GPU
3013  *              to memory
3014  *              clflush (obj) to invalidate the CPU cache
3015  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3016  *      5. Read/written by CPU
3017  *              cache lines are loaded and dirtied
3018  *      6. Read written by GPU
3019  *              Same as last GPU access
3020  *
3021  * Case 3: The constant buffer
3022  *
3023  *      1. Allocated
3024  *      2. Written by CPU
3025  *      3. Read by GPU
3026  *      4. Updated (written) by CPU again
3027  *      5. Read by GPU
3028  *
3029  *      1. Allocated
3030  *              (CPU, CPU)
3031  *      2. Written by CPU
3032  *              (CPU, CPU)
3033  *      3. Read by GPU
3034  *              (CPU+RENDER, 0)
3035  *              flush_domains = CPU
3036  *              invalidate_domains = RENDER
3037  *              clflush (obj)
3038  *              MI_FLUSH
3039  *              drm_agp_chipset_flush
3040  *      4. Updated (written) by CPU again
3041  *              (CPU, CPU)
3042  *              flush_domains = 0 (no previous write domain)
3043  *              invalidate_domains = 0 (no new read domains)
3044  *      5. Read by GPU
3045  *              (CPU+RENDER, 0)
3046  *              flush_domains = CPU
3047  *              invalidate_domains = RENDER
3048  *              clflush (obj)
3049  *              MI_FLUSH
3050  *              drm_agp_chipset_flush
3051  */
3052 static void
3053 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3054 {
3055         struct drm_device               *dev = obj->dev;
3056         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
3057         uint32_t                        invalidate_domains = 0;
3058         uint32_t                        flush_domains = 0;
3059         uint32_t                        old_read_domains;
3060
3061         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3062         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
3063
3064         intel_mark_busy(dev, obj);
3065
3066 #if WATCH_BUF
3067         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3068                  __func__, obj,
3069                  obj->read_domains, obj->pending_read_domains,
3070                  obj->write_domain, obj->pending_write_domain);
3071 #endif
3072         /*
3073          * If the object isn't moving to a new write domain,
3074          * let the object stay in multiple read domains
3075          */
3076         if (obj->pending_write_domain == 0)
3077                 obj->pending_read_domains |= obj->read_domains;
3078         else
3079                 obj_priv->dirty = 1;
3080
3081         /*
3082          * Flush the current write domain if
3083          * the new read domains don't match. Invalidate
3084          * any read domains which differ from the old
3085          * write domain
3086          */
3087         if (obj->write_domain &&
3088             obj->write_domain != obj->pending_read_domains) {
3089                 flush_domains |= obj->write_domain;
3090                 invalidate_domains |=
3091                         obj->pending_read_domains & ~obj->write_domain;
3092         }
3093         /*
3094          * Invalidate any read caches which may have
3095          * stale data. That is, any new read domains.
3096          */
3097         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3098         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3099 #if WATCH_BUF
3100                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3101                          __func__, flush_domains, invalidate_domains);
3102 #endif
3103                 i915_gem_clflush_object(obj);
3104         }
3105
3106         old_read_domains = obj->read_domains;
3107
3108         /* The actual obj->write_domain will be updated with
3109          * pending_write_domain after we emit the accumulated flush for all
3110          * of our domain changes in execbuffers (which clears objects'
3111          * write_domains).  So if we have a current write domain that we
3112          * aren't changing, set pending_write_domain to that.
3113          */
3114         if (flush_domains == 0 && obj->pending_write_domain == 0)
3115                 obj->pending_write_domain = obj->write_domain;
3116         obj->read_domains = obj->pending_read_domains;
3117
3118         dev->invalidate_domains |= invalidate_domains;
3119         dev->flush_domains |= flush_domains;
3120 #if WATCH_BUF
3121         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3122                  __func__,
3123                  obj->read_domains, obj->write_domain,
3124                  dev->invalidate_domains, dev->flush_domains);
3125 #endif
3126
3127         trace_i915_gem_object_change_domain(obj,
3128                                             old_read_domains,
3129                                             obj->write_domain);
3130 }
3131
3132 /**
3133  * Moves the object from a partially CPU read to a full one.
3134  *
3135  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3136  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3137  */
3138 static void
3139 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3140 {
3141         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3142
3143         if (!obj_priv->page_cpu_valid)
3144                 return;
3145
3146         /* If we're partially in the CPU read domain, finish moving it in.
3147          */
3148         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3149                 int i;
3150
3151                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3152                         if (obj_priv->page_cpu_valid[i])
3153                                 continue;
3154                         drm_clflush_pages(obj_priv->pages + i, 1);
3155                 }
3156         }
3157
3158         /* Free the page_cpu_valid mappings which are now stale, whether
3159          * or not we've got I915_GEM_DOMAIN_CPU.
3160          */
3161         kfree(obj_priv->page_cpu_valid);
3162         obj_priv->page_cpu_valid = NULL;
3163 }
3164
3165 /**
3166  * Set the CPU read domain on a range of the object.
3167  *
3168  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3169  * not entirely valid.  The page_cpu_valid member of the object flags which
3170  * pages have been flushed, and will be respected by
3171  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3172  * of the whole object.
3173  *
3174  * This function returns when the move is complete, including waiting on
3175  * flushes to occur.
3176  */
3177 static int
3178 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3179                                           uint64_t offset, uint64_t size)
3180 {
3181         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3182         uint32_t old_read_domains;
3183         int i, ret;
3184
3185         if (offset == 0 && size == obj->size)
3186                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3187
3188         i915_gem_object_flush_gpu_write_domain(obj);
3189         /* Wait on any GPU rendering and flushing to occur. */
3190         ret = i915_gem_object_wait_rendering(obj);
3191         if (ret != 0)
3192                 return ret;
3193         i915_gem_object_flush_gtt_write_domain(obj);
3194
3195         /* If we're already fully in the CPU read domain, we're done. */
3196         if (obj_priv->page_cpu_valid == NULL &&
3197             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3198                 return 0;
3199
3200         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3201          * newly adding I915_GEM_DOMAIN_CPU
3202          */
3203         if (obj_priv->page_cpu_valid == NULL) {
3204                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3205                                                    GFP_KERNEL);
3206                 if (obj_priv->page_cpu_valid == NULL)
3207                         return -ENOMEM;
3208         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3209                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3210
3211         /* Flush the cache on any pages that are still invalid from the CPU's
3212          * perspective.
3213          */
3214         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3215              i++) {
3216                 if (obj_priv->page_cpu_valid[i])
3217                         continue;
3218
3219                 drm_clflush_pages(obj_priv->pages + i, 1);
3220
3221                 obj_priv->page_cpu_valid[i] = 1;
3222         }
3223
3224         /* It should now be out of any other write domains, and we can update
3225          * the domain values for our changes.
3226          */
3227         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3228
3229         old_read_domains = obj->read_domains;
3230         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3231
3232         trace_i915_gem_object_change_domain(obj,
3233                                             old_read_domains,
3234                                             obj->write_domain);
3235
3236         return 0;
3237 }
3238
3239 /**
3240  * Pin an object to the GTT and evaluate the relocations landing in it.
3241  */
3242 static int
3243 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3244                                  struct drm_file *file_priv,
3245                                  struct drm_i915_gem_exec_object2 *entry,
3246                                  struct drm_i915_gem_relocation_entry *relocs)
3247 {
3248         struct drm_device *dev = obj->dev;
3249         drm_i915_private_t *dev_priv = dev->dev_private;
3250         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3251         int i, ret;
3252         void __iomem *reloc_page;
3253         bool need_fence;
3254
3255         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3256                      obj_priv->tiling_mode != I915_TILING_NONE;
3257
3258         /* Check fence reg constraints and rebind if necessary */
3259         if (need_fence && !i915_gem_object_fence_offset_ok(obj,
3260             obj_priv->tiling_mode))
3261                 i915_gem_object_unbind(obj);
3262
3263         /* Choose the GTT offset for our buffer and put it there. */
3264         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3265         if (ret)
3266                 return ret;
3267
3268         /*
3269          * Pre-965 chips need a fence register set up in order to
3270          * properly handle blits to/from tiled surfaces.
3271          */
3272         if (need_fence) {
3273                 ret = i915_gem_object_get_fence_reg(obj);
3274                 if (ret != 0) {
3275                         if (ret != -EBUSY && ret != -ERESTARTSYS)
3276                                 DRM_ERROR("Failure to install fence: %d\n",
3277                                           ret);
3278                         i915_gem_object_unpin(obj);
3279                         return ret;
3280                 }
3281         }
3282
3283         entry->offset = obj_priv->gtt_offset;
3284
3285         /* Apply the relocations, using the GTT aperture to avoid cache
3286          * flushing requirements.
3287          */
3288         for (i = 0; i < entry->relocation_count; i++) {
3289                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3290                 struct drm_gem_object *target_obj;
3291                 struct drm_i915_gem_object *target_obj_priv;
3292                 uint32_t reloc_val, reloc_offset;
3293                 uint32_t __iomem *reloc_entry;
3294
3295                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3296                                                    reloc->target_handle);
3297                 if (target_obj == NULL) {
3298                         i915_gem_object_unpin(obj);
3299                         return -EBADF;
3300                 }
3301                 target_obj_priv = target_obj->driver_private;
3302
3303 #if WATCH_RELOC
3304                 DRM_INFO("%s: obj %p offset %08x target %d "
3305                          "read %08x write %08x gtt %08x "
3306                          "presumed %08x delta %08x\n",
3307                          __func__,
3308                          obj,
3309                          (int) reloc->offset,
3310                          (int) reloc->target_handle,
3311                          (int) reloc->read_domains,
3312                          (int) reloc->write_domain,
3313                          (int) target_obj_priv->gtt_offset,
3314                          (int) reloc->presumed_offset,
3315                          reloc->delta);
3316 #endif
3317
3318                 /* The target buffer should have appeared before us in the
3319                  * exec_object list, so it should have a GTT space bound by now.
3320                  */
3321                 if (target_obj_priv->gtt_space == NULL) {
3322                         DRM_ERROR("No GTT space found for object %d\n",
3323                                   reloc->target_handle);
3324                         drm_gem_object_unreference(target_obj);
3325                         i915_gem_object_unpin(obj);
3326                         return -EINVAL;
3327                 }
3328
3329                 /* Validate that the target is in a valid r/w GPU domain */
3330                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3331                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3332                         DRM_ERROR("reloc with read/write CPU domains: "
3333                                   "obj %p target %d offset %d "
3334                                   "read %08x write %08x",
3335                                   obj, reloc->target_handle,
3336                                   (int) reloc->offset,
3337                                   reloc->read_domains,
3338                                   reloc->write_domain);
3339                         drm_gem_object_unreference(target_obj);
3340                         i915_gem_object_unpin(obj);
3341                         return -EINVAL;
3342                 }
3343                 if (reloc->write_domain && target_obj->pending_write_domain &&
3344                     reloc->write_domain != target_obj->pending_write_domain) {
3345                         DRM_ERROR("Write domain conflict: "
3346                                   "obj %p target %d offset %d "
3347                                   "new %08x old %08x\n",
3348                                   obj, reloc->target_handle,
3349                                   (int) reloc->offset,
3350                                   reloc->write_domain,
3351                                   target_obj->pending_write_domain);
3352                         drm_gem_object_unreference(target_obj);
3353                         i915_gem_object_unpin(obj);
3354                         return -EINVAL;
3355                 }
3356
3357                 target_obj->pending_read_domains |= reloc->read_domains;
3358                 target_obj->pending_write_domain |= reloc->write_domain;
3359
3360                 /* If the relocation already has the right value in it, no
3361                  * more work needs to be done.
3362                  */
3363                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3364                         drm_gem_object_unreference(target_obj);
3365                         continue;
3366                 }
3367
3368                 /* Check that the relocation address is valid... */
3369                 if (reloc->offset > obj->size - 4) {
3370                         DRM_ERROR("Relocation beyond object bounds: "
3371                                   "obj %p target %d offset %d size %d.\n",
3372                                   obj, reloc->target_handle,
3373                                   (int) reloc->offset, (int) obj->size);
3374                         drm_gem_object_unreference(target_obj);
3375                         i915_gem_object_unpin(obj);
3376                         return -EINVAL;
3377                 }
3378                 if (reloc->offset & 3) {
3379                         DRM_ERROR("Relocation not 4-byte aligned: "
3380                                   "obj %p target %d offset %d.\n",
3381                                   obj, reloc->target_handle,
3382                                   (int) reloc->offset);
3383                         drm_gem_object_unreference(target_obj);
3384                         i915_gem_object_unpin(obj);
3385                         return -EINVAL;
3386                 }
3387
3388                 /* and points to somewhere within the target object. */
3389                 if (reloc->delta >= target_obj->size) {
3390                         DRM_ERROR("Relocation beyond target object bounds: "
3391                                   "obj %p target %d delta %d size %d.\n",
3392                                   obj, reloc->target_handle,
3393                                   (int) reloc->delta, (int) target_obj->size);
3394                         drm_gem_object_unreference(target_obj);
3395                         i915_gem_object_unpin(obj);
3396                         return -EINVAL;
3397                 }
3398
3399                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3400                 if (ret != 0) {
3401                         drm_gem_object_unreference(target_obj);
3402                         i915_gem_object_unpin(obj);
3403                         return -EINVAL;
3404                 }
3405
3406                 /* Map the page containing the relocation we're going to
3407                  * perform.
3408                  */
3409                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3410                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3411                                                       (reloc_offset &
3412                                                        ~(PAGE_SIZE - 1)));
3413                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3414                                                    (reloc_offset & (PAGE_SIZE - 1)));
3415                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3416
3417 #if WATCH_BUF
3418                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3419                           obj, (unsigned int) reloc->offset,
3420                           readl(reloc_entry), reloc_val);
3421 #endif
3422                 writel(reloc_val, reloc_entry);
3423                 io_mapping_unmap_atomic(reloc_page);
3424
3425                 /* The updated presumed offset for this entry will be
3426                  * copied back out to the user.
3427                  */
3428                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3429
3430                 drm_gem_object_unreference(target_obj);
3431         }
3432
3433 #if WATCH_BUF
3434         if (0)
3435                 i915_gem_dump_object(obj, 128, __func__, ~0);
3436 #endif
3437         return 0;
3438 }
3439
3440 /** Dispatch a batchbuffer to the ring
3441  */
3442 static int
3443 i915_dispatch_gem_execbuffer(struct drm_device *dev,
3444                               struct drm_i915_gem_execbuffer2 *exec,
3445                               struct drm_clip_rect *cliprects,
3446                               uint64_t exec_offset)
3447 {
3448         drm_i915_private_t *dev_priv = dev->dev_private;
3449         int nbox = exec->num_cliprects;
3450         int i = 0, count;
3451         uint32_t exec_start, exec_len;
3452         RING_LOCALS;
3453
3454         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3455         exec_len = (uint32_t) exec->batch_len;
3456
3457         trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
3458
3459         count = nbox ? nbox : 1;
3460
3461         for (i = 0; i < count; i++) {
3462                 if (i < nbox) {
3463                         int ret = i915_emit_box(dev, cliprects, i,
3464                                                 exec->DR1, exec->DR4);
3465                         if (ret)
3466                                 return ret;
3467                 }
3468
3469                 if (IS_I830(dev) || IS_845G(dev)) {
3470                         BEGIN_LP_RING(4);
3471                         OUT_RING(MI_BATCH_BUFFER);
3472                         OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3473                         OUT_RING(exec_start + exec_len - 4);
3474                         OUT_RING(0);
3475                         ADVANCE_LP_RING();
3476                 } else {
3477                         BEGIN_LP_RING(2);
3478                         if (IS_I965G(dev)) {
3479                                 OUT_RING(MI_BATCH_BUFFER_START |
3480                                          (2 << 6) |
3481                                          MI_BATCH_NON_SECURE_I965);
3482                                 OUT_RING(exec_start);
3483                         } else {
3484                                 OUT_RING(MI_BATCH_BUFFER_START |
3485                                          (2 << 6));
3486                                 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3487                         }
3488                         ADVANCE_LP_RING();
3489                 }
3490         }
3491
3492         /* XXX breadcrumb */
3493         return 0;
3494 }
3495
3496 /* Throttle our rendering by waiting until the ring has completed our requests
3497  * emitted over 20 msec ago.
3498  *
3499  * Note that if we were to use the current jiffies each time around the loop,
3500  * we wouldn't escape the function with any frames outstanding if the time to
3501  * render a frame was over 20ms.
3502  *
3503  * This should get us reasonable parallelism between CPU and GPU but also
3504  * relatively low latency when blocking on a particular request to finish.
3505  */
3506 static int
3507 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3508 {
3509         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3510         int ret = 0;
3511         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3512
3513         mutex_lock(&dev->struct_mutex);
3514         while (!list_empty(&i915_file_priv->mm.request_list)) {
3515                 struct drm_i915_gem_request *request;
3516
3517                 request = list_first_entry(&i915_file_priv->mm.request_list,
3518                                            struct drm_i915_gem_request,
3519                                            client_list);
3520
3521                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3522                         break;
3523
3524                 ret = i915_wait_request(dev, request->seqno);
3525                 if (ret != 0)
3526                         break;
3527         }
3528         mutex_unlock(&dev->struct_mutex);
3529
3530         return ret;
3531 }
3532
3533 static int
3534 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3535                               uint32_t buffer_count,
3536                               struct drm_i915_gem_relocation_entry **relocs)
3537 {
3538         uint32_t reloc_count = 0, reloc_index = 0, i;
3539         int ret;
3540
3541         *relocs = NULL;
3542         for (i = 0; i < buffer_count; i++) {
3543                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3544                         return -EINVAL;
3545                 reloc_count += exec_list[i].relocation_count;
3546         }
3547
3548         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3549         if (*relocs == NULL) {
3550                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3551                 return -ENOMEM;
3552         }
3553
3554         for (i = 0; i < buffer_count; i++) {
3555                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3556
3557                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3558
3559                 ret = copy_from_user(&(*relocs)[reloc_index],
3560                                      user_relocs,
3561                                      exec_list[i].relocation_count *
3562                                      sizeof(**relocs));
3563                 if (ret != 0) {
3564                         drm_free_large(*relocs);
3565                         *relocs = NULL;
3566                         return -EFAULT;
3567                 }
3568
3569                 reloc_index += exec_list[i].relocation_count;
3570         }
3571
3572         return 0;
3573 }
3574
3575 static int
3576 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3577                             uint32_t buffer_count,
3578                             struct drm_i915_gem_relocation_entry *relocs)
3579 {
3580         uint32_t reloc_count = 0, i;
3581         int ret = 0;
3582
3583         if (relocs == NULL)
3584             return 0;
3585
3586         for (i = 0; i < buffer_count; i++) {
3587                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3588                 int unwritten;
3589
3590                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3591
3592                 unwritten = copy_to_user(user_relocs,
3593                                          &relocs[reloc_count],
3594                                          exec_list[i].relocation_count *
3595                                          sizeof(*relocs));
3596
3597                 if (unwritten) {
3598                         ret = -EFAULT;
3599                         goto err;
3600                 }
3601
3602                 reloc_count += exec_list[i].relocation_count;
3603         }
3604
3605 err:
3606         drm_free_large(relocs);
3607
3608         return ret;
3609 }
3610
3611 static int
3612 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3613                            uint64_t exec_offset)
3614 {
3615         uint32_t exec_start, exec_len;
3616
3617         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3618         exec_len = (uint32_t) exec->batch_len;
3619
3620         if ((exec_start | exec_len) & 0x7)
3621                 return -EINVAL;
3622
3623         if (!exec_start)
3624                 return -EINVAL;
3625
3626         return 0;
3627 }
3628
3629 static int
3630 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3631                                struct drm_gem_object **object_list,
3632                                int count)
3633 {
3634         drm_i915_private_t *dev_priv = dev->dev_private;
3635         struct drm_i915_gem_object *obj_priv;
3636         DEFINE_WAIT(wait);
3637         int i, ret = 0;
3638
3639         for (;;) {
3640                 prepare_to_wait(&dev_priv->pending_flip_queue,
3641                                 &wait, TASK_INTERRUPTIBLE);
3642                 for (i = 0; i < count; i++) {
3643                         obj_priv = object_list[i]->driver_private;
3644                         if (atomic_read(&obj_priv->pending_flip) > 0)
3645                                 break;
3646                 }
3647                 if (i == count)
3648                         break;
3649
3650                 if (!signal_pending(current)) {
3651                         mutex_unlock(&dev->struct_mutex);
3652                         schedule();
3653                         mutex_lock(&dev->struct_mutex);
3654                         continue;
3655                 }
3656                 ret = -ERESTARTSYS;
3657                 break;
3658         }
3659         finish_wait(&dev_priv->pending_flip_queue, &wait);
3660
3661         return ret;
3662 }
3663
3664 int
3665 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3666                        struct drm_file *file_priv,
3667                        struct drm_i915_gem_execbuffer2 *args,
3668                        struct drm_i915_gem_exec_object2 *exec_list)
3669 {
3670         drm_i915_private_t *dev_priv = dev->dev_private;
3671         struct drm_gem_object **object_list = NULL;
3672         struct drm_gem_object *batch_obj;
3673         struct drm_i915_gem_object *obj_priv;
3674         struct drm_clip_rect *cliprects = NULL;
3675         struct drm_i915_gem_relocation_entry *relocs = NULL;
3676         int ret = 0, ret2, i, pinned = 0;
3677         uint64_t exec_offset;
3678         uint32_t seqno, flush_domains, reloc_index;
3679         int pin_tries, flips;
3680
3681 #if WATCH_EXEC
3682         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3683                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3684 #endif
3685
3686         if (args->buffer_count < 1) {
3687                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3688                 return -EINVAL;
3689         }
3690         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3691         if (object_list == NULL) {
3692                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3693                           args->buffer_count);
3694                 ret = -ENOMEM;
3695                 goto pre_mutex_err;
3696         }
3697
3698         if (args->num_cliprects != 0) {
3699                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3700                                     GFP_KERNEL);
3701                 if (cliprects == NULL) {
3702                         ret = -ENOMEM;
3703                         goto pre_mutex_err;
3704                 }
3705
3706                 ret = copy_from_user(cliprects,
3707                                      (struct drm_clip_rect __user *)
3708                                      (uintptr_t) args->cliprects_ptr,
3709                                      sizeof(*cliprects) * args->num_cliprects);
3710                 if (ret != 0) {
3711                         DRM_ERROR("copy %d cliprects failed: %d\n",
3712                                   args->num_cliprects, ret);
3713                         goto pre_mutex_err;
3714                 }
3715         }
3716
3717         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3718                                             &relocs);
3719         if (ret != 0)
3720                 goto pre_mutex_err;
3721
3722         mutex_lock(&dev->struct_mutex);
3723
3724         i915_verify_inactive(dev, __FILE__, __LINE__);
3725
3726         if (atomic_read(&dev_priv->mm.wedged)) {
3727                 mutex_unlock(&dev->struct_mutex);
3728                 ret = -EIO;
3729                 goto pre_mutex_err;
3730         }
3731
3732         if (dev_priv->mm.suspended) {
3733                 mutex_unlock(&dev->struct_mutex);
3734                 ret = -EBUSY;
3735                 goto pre_mutex_err;
3736         }
3737
3738         /* Look up object handles */
3739         flips = 0;
3740         for (i = 0; i < args->buffer_count; i++) {
3741                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3742                                                        exec_list[i].handle);
3743                 if (object_list[i] == NULL) {
3744                         DRM_ERROR("Invalid object handle %d at index %d\n",
3745                                    exec_list[i].handle, i);
3746                         /* prevent error path from reading uninitialized data */
3747                         args->buffer_count = i + 1;
3748                         ret = -EBADF;
3749                         goto err;
3750                 }
3751
3752                 obj_priv = object_list[i]->driver_private;
3753                 if (obj_priv->in_execbuffer) {
3754                         DRM_ERROR("Object %p appears more than once in object list\n",
3755                                    object_list[i]);
3756                         /* prevent error path from reading uninitialized data */
3757                         args->buffer_count = i + 1;
3758                         ret = -EBADF;
3759                         goto err;
3760                 }
3761                 obj_priv->in_execbuffer = true;
3762                 flips += atomic_read(&obj_priv->pending_flip);
3763         }
3764
3765         if (flips > 0) {
3766                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3767                                                      args->buffer_count);
3768                 if (ret)
3769                         goto err;
3770         }
3771
3772         /* Pin and relocate */
3773         for (pin_tries = 0; ; pin_tries++) {
3774                 ret = 0;
3775                 reloc_index = 0;
3776
3777                 for (i = 0; i < args->buffer_count; i++) {
3778                         object_list[i]->pending_read_domains = 0;
3779                         object_list[i]->pending_write_domain = 0;
3780                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3781                                                                file_priv,
3782                                                                &exec_list[i],
3783                                                                &relocs[reloc_index]);
3784                         if (ret)
3785                                 break;
3786                         pinned = i + 1;
3787                         reloc_index += exec_list[i].relocation_count;
3788                 }
3789                 /* success */
3790                 if (ret == 0)
3791                         break;
3792
3793                 /* error other than GTT full, or we've already tried again */
3794                 if (ret != -ENOSPC || pin_tries >= 1) {
3795                         if (ret != -ERESTARTSYS) {
3796                                 unsigned long long total_size = 0;
3797                                 for (i = 0; i < args->buffer_count; i++)
3798                                         total_size += object_list[i]->size;
3799                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes: %d\n",
3800                                           pinned+1, args->buffer_count,
3801                                           total_size, ret);
3802                                 DRM_ERROR("%d objects [%d pinned], "
3803                                           "%d object bytes [%d pinned], "
3804                                           "%d/%d gtt bytes\n",
3805                                           atomic_read(&dev->object_count),
3806                                           atomic_read(&dev->pin_count),
3807                                           atomic_read(&dev->object_memory),
3808                                           atomic_read(&dev->pin_memory),
3809                                           atomic_read(&dev->gtt_memory),
3810                                           dev->gtt_total);
3811                         }
3812                         goto err;
3813                 }
3814
3815                 /* unpin all of our buffers */
3816                 for (i = 0; i < pinned; i++)
3817                         i915_gem_object_unpin(object_list[i]);
3818                 pinned = 0;
3819
3820                 /* evict everyone we can from the aperture */
3821                 ret = i915_gem_evict_everything(dev);
3822                 if (ret && ret != -ENOSPC)
3823                         goto err;
3824         }
3825
3826         /* Set the pending read domains for the batch buffer to COMMAND */
3827         batch_obj = object_list[args->buffer_count-1];
3828         if (batch_obj->pending_write_domain) {
3829                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3830                 ret = -EINVAL;
3831                 goto err;
3832         }
3833         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3834
3835         /* Sanity check the batch buffer, prior to moving objects */
3836         exec_offset = exec_list[args->buffer_count - 1].offset;
3837         ret = i915_gem_check_execbuffer (args, exec_offset);
3838         if (ret != 0) {
3839                 DRM_ERROR("execbuf with invalid offset/length\n");
3840                 goto err;
3841         }
3842
3843         i915_verify_inactive(dev, __FILE__, __LINE__);
3844
3845         /* Zero the global flush/invalidate flags. These
3846          * will be modified as new domains are computed
3847          * for each object
3848          */
3849         dev->invalidate_domains = 0;
3850         dev->flush_domains = 0;
3851
3852         for (i = 0; i < args->buffer_count; i++) {
3853                 struct drm_gem_object *obj = object_list[i];
3854
3855                 /* Compute new gpu domains and update invalidate/flush */
3856                 i915_gem_object_set_to_gpu_domain(obj);
3857         }
3858
3859         i915_verify_inactive(dev, __FILE__, __LINE__);
3860
3861         if (dev->invalidate_domains | dev->flush_domains) {
3862 #if WATCH_EXEC
3863                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3864                           __func__,
3865                          dev->invalidate_domains,
3866                          dev->flush_domains);
3867 #endif
3868                 i915_gem_flush(dev,
3869                                dev->invalidate_domains,
3870                                dev->flush_domains);
3871                 if (dev->flush_domains & I915_GEM_GPU_DOMAINS)
3872                         (void)i915_add_request(dev, file_priv,
3873                                                dev->flush_domains);
3874         }
3875
3876         for (i = 0; i < args->buffer_count; i++) {
3877                 struct drm_gem_object *obj = object_list[i];
3878                 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3879                 uint32_t old_write_domain = obj->write_domain;
3880
3881                 obj->write_domain = obj->pending_write_domain;
3882                 if (obj->write_domain)
3883                         list_move_tail(&obj_priv->gpu_write_list,
3884                                        &dev_priv->mm.gpu_write_list);
3885                 else
3886                         list_del_init(&obj_priv->gpu_write_list);
3887
3888                 trace_i915_gem_object_change_domain(obj,
3889                                                     obj->read_domains,
3890                                                     old_write_domain);
3891         }
3892
3893         i915_verify_inactive(dev, __FILE__, __LINE__);
3894
3895 #if WATCH_COHERENCY
3896         for (i = 0; i < args->buffer_count; i++) {
3897                 i915_gem_object_check_coherency(object_list[i],
3898                                                 exec_list[i].handle);
3899         }
3900 #endif
3901
3902 #if WATCH_EXEC
3903         i915_gem_dump_object(batch_obj,
3904                               args->batch_len,
3905                               __func__,
3906                               ~0);
3907 #endif
3908
3909         /* Exec the batchbuffer */
3910         ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
3911         if (ret) {
3912                 DRM_ERROR("dispatch failed %d\n", ret);
3913                 goto err;
3914         }
3915
3916         /*
3917          * Ensure that the commands in the batch buffer are
3918          * finished before the interrupt fires
3919          */
3920         flush_domains = i915_retire_commands(dev);
3921
3922         i915_verify_inactive(dev, __FILE__, __LINE__);
3923
3924         /*
3925          * Get a seqno representing the execution of the current buffer,
3926          * which we can wait on.  We would like to mitigate these interrupts,
3927          * likely by only creating seqnos occasionally (so that we have
3928          * *some* interrupts representing completion of buffers that we can
3929          * wait on when trying to clear up gtt space).
3930          */
3931         seqno = i915_add_request(dev, file_priv, flush_domains);
3932         BUG_ON(seqno == 0);
3933         for (i = 0; i < args->buffer_count; i++) {
3934                 struct drm_gem_object *obj = object_list[i];
3935
3936                 i915_gem_object_move_to_active(obj, seqno);
3937 #if WATCH_LRU
3938                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3939 #endif
3940         }
3941 #if WATCH_LRU
3942         i915_dump_lru(dev, __func__);
3943 #endif
3944
3945         i915_verify_inactive(dev, __FILE__, __LINE__);
3946
3947 err:
3948         for (i = 0; i < pinned; i++)
3949                 i915_gem_object_unpin(object_list[i]);
3950
3951         for (i = 0; i < args->buffer_count; i++) {
3952                 if (object_list[i]) {
3953                         obj_priv = object_list[i]->driver_private;
3954                         obj_priv->in_execbuffer = false;
3955                 }
3956                 drm_gem_object_unreference(object_list[i]);
3957         }
3958
3959         mutex_unlock(&dev->struct_mutex);
3960
3961 pre_mutex_err:
3962         /* Copy the updated relocations out regardless of current error
3963          * state.  Failure to update the relocs would mean that the next
3964          * time userland calls execbuf, it would do so with presumed offset
3965          * state that didn't match the actual object state.
3966          */
3967         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3968                                            relocs);
3969         if (ret2 != 0) {
3970                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3971
3972                 if (ret == 0)
3973                         ret = ret2;
3974         }
3975
3976         drm_free_large(object_list);
3977         kfree(cliprects);
3978
3979         return ret;
3980 }
3981
3982 /*
3983  * Legacy execbuffer just creates an exec2 list from the original exec object
3984  * list array and passes it to the real function.
3985  */
3986 int
3987 i915_gem_execbuffer(struct drm_device *dev, void *data,
3988                     struct drm_file *file_priv)
3989 {
3990         struct drm_i915_gem_execbuffer *args = data;
3991         struct drm_i915_gem_execbuffer2 exec2;
3992         struct drm_i915_gem_exec_object *exec_list = NULL;
3993         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3994         int ret, i;
3995
3996 #if WATCH_EXEC
3997         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3998                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3999 #endif
4000
4001         if (args->buffer_count < 1) {
4002                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4003                 return -EINVAL;
4004         }
4005
4006         /* Copy in the exec list from userland */
4007         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4008         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4009         if (exec_list == NULL || exec2_list == NULL) {
4010                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4011                           args->buffer_count);
4012                 drm_free_large(exec_list);
4013                 drm_free_large(exec2_list);
4014                 return -ENOMEM;
4015         }
4016         ret = copy_from_user(exec_list,
4017                              (struct drm_i915_relocation_entry __user *)
4018                              (uintptr_t) args->buffers_ptr,
4019                              sizeof(*exec_list) * args->buffer_count);
4020         if (ret != 0) {
4021                 DRM_ERROR("copy %d exec entries failed %d\n",
4022                           args->buffer_count, ret);
4023                 drm_free_large(exec_list);
4024                 drm_free_large(exec2_list);
4025                 return -EFAULT;
4026         }
4027
4028         for (i = 0; i < args->buffer_count; i++) {
4029                 exec2_list[i].handle = exec_list[i].handle;
4030                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4031                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4032                 exec2_list[i].alignment = exec_list[i].alignment;
4033                 exec2_list[i].offset = exec_list[i].offset;
4034                 if (!IS_I965G(dev))
4035                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4036                 else
4037                         exec2_list[i].flags = 0;
4038         }
4039
4040         exec2.buffers_ptr = args->buffers_ptr;
4041         exec2.buffer_count = args->buffer_count;
4042         exec2.batch_start_offset = args->batch_start_offset;
4043         exec2.batch_len = args->batch_len;
4044         exec2.DR1 = args->DR1;
4045         exec2.DR4 = args->DR4;
4046         exec2.num_cliprects = args->num_cliprects;
4047         exec2.cliprects_ptr = args->cliprects_ptr;
4048         exec2.flags = 0;
4049
4050         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4051         if (!ret) {
4052                 /* Copy the new buffer offsets back to the user's exec list. */
4053                 for (i = 0; i < args->buffer_count; i++)
4054                         exec_list[i].offset = exec2_list[i].offset;
4055                 /* ... and back out to userspace */
4056                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4057                                    (uintptr_t) args->buffers_ptr,
4058                                    exec_list,
4059                                    sizeof(*exec_list) * args->buffer_count);
4060                 if (ret) {
4061                         ret = -EFAULT;
4062                         DRM_ERROR("failed to copy %d exec entries "
4063                                   "back to user (%d)\n",
4064                                   args->buffer_count, ret);
4065                 }
4066         }
4067
4068         drm_free_large(exec_list);
4069         drm_free_large(exec2_list);
4070         return ret;
4071 }
4072
4073 int
4074 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4075                      struct drm_file *file_priv)
4076 {
4077         struct drm_i915_gem_execbuffer2 *args = data;
4078         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4079         int ret;
4080
4081 #if WATCH_EXEC
4082         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4083                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4084 #endif
4085
4086         if (args->buffer_count < 1) {
4087                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4088                 return -EINVAL;
4089         }
4090
4091         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4092         if (exec2_list == NULL) {
4093                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4094                           args->buffer_count);
4095                 return -ENOMEM;
4096         }
4097         ret = copy_from_user(exec2_list,
4098                              (struct drm_i915_relocation_entry __user *)
4099                              (uintptr_t) args->buffers_ptr,
4100                              sizeof(*exec2_list) * args->buffer_count);
4101         if (ret != 0) {
4102                 DRM_ERROR("copy %d exec entries failed %d\n",
4103                           args->buffer_count, ret);
4104                 drm_free_large(exec2_list);
4105                 return -EFAULT;
4106         }
4107
4108         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4109         if (!ret) {
4110                 /* Copy the new buffer offsets back to the user's exec list. */
4111                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4112                                    (uintptr_t) args->buffers_ptr,
4113                                    exec2_list,
4114                                    sizeof(*exec2_list) * args->buffer_count);
4115                 if (ret) {
4116                         ret = -EFAULT;
4117                         DRM_ERROR("failed to copy %d exec entries "
4118                                   "back to user (%d)\n",
4119                                   args->buffer_count, ret);
4120                 }
4121         }
4122
4123         drm_free_large(exec2_list);
4124         return ret;
4125 }
4126
4127 int
4128 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4129 {
4130         struct drm_device *dev = obj->dev;
4131         struct drm_i915_gem_object *obj_priv = obj->driver_private;
4132         int ret;
4133
4134         i915_verify_inactive(dev, __FILE__, __LINE__);
4135         if (obj_priv->gtt_space == NULL) {
4136                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4137                 if (ret)
4138                         return ret;
4139         }
4140
4141         obj_priv->pin_count++;
4142
4143         /* If the object is not active and not pending a flush,
4144          * remove it from the inactive list
4145          */
4146         if (obj_priv->pin_count == 1) {
4147                 atomic_inc(&dev->pin_count);
4148                 atomic_add(obj->size, &dev->pin_memory);
4149                 if (!obj_priv->active &&
4150                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
4151                     !list_empty(&obj_priv->list))
4152                         list_del_init(&obj_priv->list);
4153         }
4154         i915_verify_inactive(dev, __FILE__, __LINE__);
4155
4156         return 0;
4157 }
4158
4159 void
4160 i915_gem_object_unpin(struct drm_gem_object *obj)
4161 {
4162         struct drm_device *dev = obj->dev;
4163         drm_i915_private_t *dev_priv = dev->dev_private;
4164         struct drm_i915_gem_object *obj_priv = obj->driver_private;
4165
4166         i915_verify_inactive(dev, __FILE__, __LINE__);
4167         obj_priv->pin_count--;
4168         BUG_ON(obj_priv->pin_count < 0);
4169         BUG_ON(obj_priv->gtt_space == NULL);
4170
4171         /* If the object is no longer pinned, and is
4172          * neither active nor being flushed, then stick it on
4173          * the inactive list
4174          */
4175         if (obj_priv->pin_count == 0) {
4176                 if (!obj_priv->active &&
4177                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4178                         list_move_tail(&obj_priv->list,
4179                                        &dev_priv->mm.inactive_list);
4180                 atomic_dec(&dev->pin_count);
4181                 atomic_sub(obj->size, &dev->pin_memory);
4182         }
4183         i915_verify_inactive(dev, __FILE__, __LINE__);
4184 }
4185
4186 int
4187 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4188                    struct drm_file *file_priv)
4189 {
4190         struct drm_i915_gem_pin *args = data;
4191         struct drm_gem_object *obj;
4192         struct drm_i915_gem_object *obj_priv;
4193         int ret;
4194
4195         mutex_lock(&dev->struct_mutex);
4196
4197         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4198         if (obj == NULL) {
4199                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4200                           args->handle);
4201                 mutex_unlock(&dev->struct_mutex);
4202                 return -EBADF;
4203         }
4204         obj_priv = obj->driver_private;
4205
4206         if (obj_priv->madv != I915_MADV_WILLNEED) {
4207                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4208                 drm_gem_object_unreference(obj);
4209                 mutex_unlock(&dev->struct_mutex);
4210                 return -EINVAL;
4211         }
4212
4213         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4214                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4215                           args->handle);
4216                 drm_gem_object_unreference(obj);
4217                 mutex_unlock(&dev->struct_mutex);
4218                 return -EINVAL;
4219         }
4220
4221         obj_priv->user_pin_count++;
4222         obj_priv->pin_filp = file_priv;
4223         if (obj_priv->user_pin_count == 1) {
4224                 ret = i915_gem_object_pin(obj, args->alignment);
4225                 if (ret != 0) {
4226                         drm_gem_object_unreference(obj);
4227                         mutex_unlock(&dev->struct_mutex);
4228                         return ret;
4229                 }
4230         }
4231
4232         /* XXX - flush the CPU caches for pinned objects
4233          * as the X server doesn't manage domains yet
4234          */
4235         i915_gem_object_flush_cpu_write_domain(obj);
4236         args->offset = obj_priv->gtt_offset;
4237         drm_gem_object_unreference(obj);
4238         mutex_unlock(&dev->struct_mutex);
4239
4240         return 0;
4241 }
4242
4243 int
4244 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4245                      struct drm_file *file_priv)
4246 {
4247         struct drm_i915_gem_pin *args = data;
4248         struct drm_gem_object *obj;
4249         struct drm_i915_gem_object *obj_priv;
4250
4251         mutex_lock(&dev->struct_mutex);
4252
4253         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4254         if (obj == NULL) {
4255                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4256                           args->handle);
4257                 mutex_unlock(&dev->struct_mutex);
4258                 return -EBADF;
4259         }
4260
4261         obj_priv = obj->driver_private;
4262         if (obj_priv->pin_filp != file_priv) {
4263                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4264                           args->handle);
4265                 drm_gem_object_unreference(obj);
4266                 mutex_unlock(&dev->struct_mutex);
4267                 return -EINVAL;
4268         }
4269         obj_priv->user_pin_count--;
4270         if (obj_priv->user_pin_count == 0) {
4271                 obj_priv->pin_filp = NULL;
4272                 i915_gem_object_unpin(obj);
4273         }
4274
4275         drm_gem_object_unreference(obj);
4276         mutex_unlock(&dev->struct_mutex);
4277         return 0;
4278 }
4279
4280 int
4281 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4282                     struct drm_file *file_priv)
4283 {
4284         struct drm_i915_gem_busy *args = data;
4285         struct drm_gem_object *obj;
4286         struct drm_i915_gem_object *obj_priv;
4287
4288         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4289         if (obj == NULL) {
4290                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4291                           args->handle);
4292                 return -EBADF;
4293         }
4294
4295         mutex_lock(&dev->struct_mutex);
4296         /* Update the active list for the hardware's current position.
4297          * Otherwise this only updates on a delayed timer or when irqs are
4298          * actually unmasked, and our working set ends up being larger than
4299          * required.
4300          */
4301         i915_gem_retire_requests(dev);
4302
4303         obj_priv = obj->driver_private;
4304         /* Don't count being on the flushing list against the object being
4305          * done.  Otherwise, a buffer left on the flushing list but not getting
4306          * flushed (because nobody's flushing that domain) won't ever return
4307          * unbusy and get reused by libdrm's bo cache.  The other expected
4308          * consumer of this interface, OpenGL's occlusion queries, also specs
4309          * that the objects get unbusy "eventually" without any interference.
4310          */
4311         args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
4312
4313         drm_gem_object_unreference(obj);
4314         mutex_unlock(&dev->struct_mutex);
4315         return 0;
4316 }
4317
4318 int
4319 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4320                         struct drm_file *file_priv)
4321 {
4322     return i915_gem_ring_throttle(dev, file_priv);
4323 }
4324
4325 int
4326 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4327                        struct drm_file *file_priv)
4328 {
4329         struct drm_i915_gem_madvise *args = data;
4330         struct drm_gem_object *obj;
4331         struct drm_i915_gem_object *obj_priv;
4332
4333         switch (args->madv) {
4334         case I915_MADV_DONTNEED:
4335         case I915_MADV_WILLNEED:
4336             break;
4337         default:
4338             return -EINVAL;
4339         }
4340
4341         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4342         if (obj == NULL) {
4343                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4344                           args->handle);
4345                 return -EBADF;
4346         }
4347
4348         mutex_lock(&dev->struct_mutex);
4349         obj_priv = obj->driver_private;
4350
4351         if (obj_priv->pin_count) {
4352                 drm_gem_object_unreference(obj);
4353                 mutex_unlock(&dev->struct_mutex);
4354
4355                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4356                 return -EINVAL;
4357         }
4358
4359         if (obj_priv->madv != __I915_MADV_PURGED)
4360                 obj_priv->madv = args->madv;
4361
4362         /* if the object is no longer bound, discard its backing storage */
4363         if (i915_gem_object_is_purgeable(obj_priv) &&
4364             obj_priv->gtt_space == NULL)
4365                 i915_gem_object_truncate(obj);
4366
4367         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4368
4369         drm_gem_object_unreference(obj);
4370         mutex_unlock(&dev->struct_mutex);
4371
4372         return 0;
4373 }
4374
4375 int i915_gem_init_object(struct drm_gem_object *obj)
4376 {
4377         struct drm_i915_gem_object *obj_priv;
4378
4379         obj_priv = kzalloc(sizeof(*obj_priv), GFP_KERNEL);
4380         if (obj_priv == NULL)
4381                 return -ENOMEM;
4382
4383         /*
4384          * We've just allocated pages from the kernel,
4385          * so they've just been written by the CPU with
4386          * zeros. They'll need to be clflushed before we
4387          * use them with the GPU.
4388          */
4389         obj->write_domain = I915_GEM_DOMAIN_CPU;
4390         obj->read_domains = I915_GEM_DOMAIN_CPU;
4391
4392         obj_priv->agp_type = AGP_USER_MEMORY;
4393
4394         obj->driver_private = obj_priv;
4395         obj_priv->obj = obj;
4396         obj_priv->fence_reg = I915_FENCE_REG_NONE;
4397         INIT_LIST_HEAD(&obj_priv->list);
4398         INIT_LIST_HEAD(&obj_priv->gpu_write_list);
4399         INIT_LIST_HEAD(&obj_priv->fence_list);
4400         obj_priv->madv = I915_MADV_WILLNEED;
4401
4402         trace_i915_gem_object_create(obj);
4403
4404         return 0;
4405 }
4406
4407 void i915_gem_free_object(struct drm_gem_object *obj)
4408 {
4409         struct drm_device *dev = obj->dev;
4410         struct drm_i915_gem_object *obj_priv = obj->driver_private;
4411
4412         trace_i915_gem_object_destroy(obj);
4413
4414         while (obj_priv->pin_count > 0)
4415                 i915_gem_object_unpin(obj);
4416
4417         if (obj_priv->phys_obj)
4418                 i915_gem_detach_phys_object(dev, obj);
4419
4420         i915_gem_object_unbind(obj);
4421
4422         if (obj_priv->mmap_offset)
4423                 i915_gem_free_mmap_offset(obj);
4424
4425         kfree(obj_priv->page_cpu_valid);
4426         kfree(obj_priv->bit_17);
4427         kfree(obj->driver_private);
4428 }
4429
4430 /** Unbinds all inactive objects. */
4431 static int
4432 i915_gem_evict_from_inactive_list(struct drm_device *dev)
4433 {
4434         drm_i915_private_t *dev_priv = dev->dev_private;
4435
4436         while (!list_empty(&dev_priv->mm.inactive_list)) {
4437                 struct drm_gem_object *obj;
4438                 int ret;
4439
4440                 obj = list_first_entry(&dev_priv->mm.inactive_list,
4441                                        struct drm_i915_gem_object,
4442                                        list)->obj;
4443
4444                 ret = i915_gem_object_unbind(obj);
4445                 if (ret != 0) {
4446                         DRM_ERROR("Error unbinding object: %d\n", ret);
4447                         return ret;
4448                 }
4449         }
4450
4451         return 0;
4452 }
4453
4454 static int
4455 i915_gpu_idle(struct drm_device *dev)
4456 {
4457         drm_i915_private_t *dev_priv = dev->dev_private;
4458         bool lists_empty;
4459         uint32_t seqno;
4460
4461         spin_lock(&dev_priv->mm.active_list_lock);
4462         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4463                       list_empty(&dev_priv->mm.active_list);
4464         spin_unlock(&dev_priv->mm.active_list_lock);
4465
4466         if (lists_empty)
4467                 return 0;
4468
4469         /* Flush everything onto the inactive list. */
4470         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
4471         seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
4472         if (seqno == 0)
4473                 return -ENOMEM;
4474
4475         return i915_wait_request(dev, seqno);
4476 }
4477
4478 int
4479 i915_gem_idle(struct drm_device *dev)
4480 {
4481         drm_i915_private_t *dev_priv = dev->dev_private;
4482         int ret;
4483
4484         mutex_lock(&dev->struct_mutex);
4485
4486         if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
4487                 mutex_unlock(&dev->struct_mutex);
4488                 return 0;
4489         }
4490
4491         ret = i915_gpu_idle(dev);
4492         if (ret) {
4493                 mutex_unlock(&dev->struct_mutex);
4494                 return ret;
4495         }
4496
4497         /* Under UMS, be paranoid and evict. */
4498         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4499                 ret = i915_gem_evict_from_inactive_list(dev);
4500                 if (ret) {
4501                         mutex_unlock(&dev->struct_mutex);
4502                         return ret;
4503                 }
4504         }
4505
4506         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4507          * We need to replace this with a semaphore, or something.
4508          * And not confound mm.suspended!
4509          */
4510         dev_priv->mm.suspended = 1;
4511         del_timer(&dev_priv->hangcheck_timer);
4512
4513         i915_kernel_lost_context(dev);
4514         i915_gem_cleanup_ringbuffer(dev);
4515
4516         mutex_unlock(&dev->struct_mutex);
4517
4518         /* Cancel the retire work handler, which should be idle now. */
4519         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4520
4521         return 0;
4522 }
4523
4524 static int
4525 i915_gem_init_hws(struct drm_device *dev)
4526 {
4527         drm_i915_private_t *dev_priv = dev->dev_private;
4528         struct drm_gem_object *obj;
4529         struct drm_i915_gem_object *obj_priv;
4530         int ret;
4531
4532         /* If we need a physical address for the status page, it's already
4533          * initialized at driver load time.
4534          */
4535         if (!I915_NEED_GFX_HWS(dev))
4536                 return 0;
4537
4538         obj = drm_gem_object_alloc(dev, 4096);
4539         if (obj == NULL) {
4540                 DRM_ERROR("Failed to allocate status page\n");
4541                 return -ENOMEM;
4542         }
4543         obj_priv = obj->driver_private;
4544         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4545
4546         ret = i915_gem_object_pin(obj, 4096);
4547         if (ret != 0) {
4548                 drm_gem_object_unreference(obj);
4549                 return ret;
4550         }
4551
4552         dev_priv->status_gfx_addr = obj_priv->gtt_offset;
4553
4554         dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
4555         if (dev_priv->hw_status_page == NULL) {
4556                 DRM_ERROR("Failed to map status page.\n");
4557                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4558                 i915_gem_object_unpin(obj);
4559                 drm_gem_object_unreference(obj);
4560                 return -EINVAL;
4561         }
4562         dev_priv->hws_obj = obj;
4563         memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
4564         if (IS_GEN6(dev)) {
4565                 I915_WRITE(HWS_PGA_GEN6, dev_priv->status_gfx_addr);
4566                 I915_READ(HWS_PGA_GEN6); /* posting read */
4567         } else {
4568                 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
4569                 I915_READ(HWS_PGA); /* posting read */
4570         }
4571         DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
4572
4573         return 0;
4574 }
4575
4576 static void
4577 i915_gem_cleanup_hws(struct drm_device *dev)
4578 {
4579         drm_i915_private_t *dev_priv = dev->dev_private;
4580         struct drm_gem_object *obj;
4581         struct drm_i915_gem_object *obj_priv;
4582
4583         if (dev_priv->hws_obj == NULL)
4584                 return;
4585
4586         obj = dev_priv->hws_obj;
4587         obj_priv = obj->driver_private;
4588
4589         kunmap(obj_priv->pages[0]);
4590         i915_gem_object_unpin(obj);
4591         drm_gem_object_unreference(obj);
4592         dev_priv->hws_obj = NULL;
4593
4594         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4595         dev_priv->hw_status_page = NULL;
4596
4597         /* Write high address into HWS_PGA when disabling. */
4598         I915_WRITE(HWS_PGA, 0x1ffff000);
4599 }
4600
4601 int
4602 i915_gem_init_ringbuffer(struct drm_device *dev)
4603 {
4604         drm_i915_private_t *dev_priv = dev->dev_private;
4605         struct drm_gem_object *obj;
4606         struct drm_i915_gem_object *obj_priv;
4607         drm_i915_ring_buffer_t *ring = &dev_priv->ring;
4608         int ret;
4609         u32 head;
4610
4611         ret = i915_gem_init_hws(dev);
4612         if (ret != 0)
4613                 return ret;
4614
4615         obj = drm_gem_object_alloc(dev, 128 * 1024);
4616         if (obj == NULL) {
4617                 DRM_ERROR("Failed to allocate ringbuffer\n");
4618                 i915_gem_cleanup_hws(dev);
4619                 return -ENOMEM;
4620         }
4621         obj_priv = obj->driver_private;
4622
4623         ret = i915_gem_object_pin(obj, 4096);
4624         if (ret != 0) {
4625                 drm_gem_object_unreference(obj);
4626                 i915_gem_cleanup_hws(dev);
4627                 return ret;
4628         }
4629
4630         /* Set up the kernel mapping for the ring. */
4631         ring->Size = obj->size;
4632
4633         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4634         ring->map.size = obj->size;
4635         ring->map.type = 0;
4636         ring->map.flags = 0;
4637         ring->map.mtrr = 0;
4638
4639         drm_core_ioremap_wc(&ring->map, dev);
4640         if (ring->map.handle == NULL) {
4641                 DRM_ERROR("Failed to map ringbuffer.\n");
4642                 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4643                 i915_gem_object_unpin(obj);
4644                 drm_gem_object_unreference(obj);
4645                 i915_gem_cleanup_hws(dev);
4646                 return -EINVAL;
4647         }
4648         ring->ring_obj = obj;
4649         ring->virtual_start = ring->map.handle;
4650
4651         /* Stop the ring if it's running. */
4652         I915_WRITE(PRB0_CTL, 0);
4653         I915_WRITE(PRB0_TAIL, 0);
4654         I915_WRITE(PRB0_HEAD, 0);
4655
4656         /* Initialize the ring. */
4657         I915_WRITE(PRB0_START, obj_priv->gtt_offset);
4658         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4659
4660         /* G45 ring initialization fails to reset head to zero */
4661         if (head != 0) {
4662                 DRM_ERROR("Ring head not reset to zero "
4663                           "ctl %08x head %08x tail %08x start %08x\n",
4664                           I915_READ(PRB0_CTL),
4665                           I915_READ(PRB0_HEAD),
4666                           I915_READ(PRB0_TAIL),
4667                           I915_READ(PRB0_START));
4668                 I915_WRITE(PRB0_HEAD, 0);
4669
4670                 DRM_ERROR("Ring head forced to zero "
4671                           "ctl %08x head %08x tail %08x start %08x\n",
4672                           I915_READ(PRB0_CTL),
4673                           I915_READ(PRB0_HEAD),
4674                           I915_READ(PRB0_TAIL),
4675                           I915_READ(PRB0_START));
4676         }
4677
4678         I915_WRITE(PRB0_CTL,
4679                    ((obj->size - 4096) & RING_NR_PAGES) |
4680                    RING_NO_REPORT |
4681                    RING_VALID);
4682
4683         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4684
4685         /* If the head is still not zero, the ring is dead */
4686         if (head != 0) {
4687                 DRM_ERROR("Ring initialization failed "
4688                           "ctl %08x head %08x tail %08x start %08x\n",
4689                           I915_READ(PRB0_CTL),
4690                           I915_READ(PRB0_HEAD),
4691                           I915_READ(PRB0_TAIL),
4692                           I915_READ(PRB0_START));
4693                 return -EIO;
4694         }
4695
4696         /* Update our cache of the ring state */
4697         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4698                 i915_kernel_lost_context(dev);
4699         else {
4700                 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4701                 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4702                 ring->space = ring->head - (ring->tail + 8);
4703                 if (ring->space < 0)
4704                         ring->space += ring->Size;
4705         }
4706
4707         return 0;
4708 }
4709
4710 void
4711 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4712 {
4713         drm_i915_private_t *dev_priv = dev->dev_private;
4714
4715         if (dev_priv->ring.ring_obj == NULL)
4716                 return;
4717
4718         drm_core_ioremapfree(&dev_priv->ring.map, dev);
4719
4720         i915_gem_object_unpin(dev_priv->ring.ring_obj);
4721         drm_gem_object_unreference(dev_priv->ring.ring_obj);
4722         dev_priv->ring.ring_obj = NULL;
4723         memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4724
4725         i915_gem_cleanup_hws(dev);
4726 }
4727
4728 int
4729 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4730                        struct drm_file *file_priv)
4731 {
4732         drm_i915_private_t *dev_priv = dev->dev_private;
4733         int ret;
4734
4735         if (drm_core_check_feature(dev, DRIVER_MODESET))
4736                 return 0;
4737
4738         if (atomic_read(&dev_priv->mm.wedged)) {
4739                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4740                 atomic_set(&dev_priv->mm.wedged, 0);
4741         }
4742
4743         mutex_lock(&dev->struct_mutex);
4744         dev_priv->mm.suspended = 0;
4745
4746         ret = i915_gem_init_ringbuffer(dev);
4747         if (ret != 0) {
4748                 mutex_unlock(&dev->struct_mutex);
4749                 return ret;
4750         }
4751
4752         spin_lock(&dev_priv->mm.active_list_lock);
4753         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4754         spin_unlock(&dev_priv->mm.active_list_lock);
4755
4756         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4757         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4758         BUG_ON(!list_empty(&dev_priv->mm.request_list));
4759         mutex_unlock(&dev->struct_mutex);
4760
4761         drm_irq_install(dev);
4762
4763         return 0;
4764 }
4765
4766 int
4767 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4768                        struct drm_file *file_priv)
4769 {
4770         if (drm_core_check_feature(dev, DRIVER_MODESET))
4771                 return 0;
4772
4773         drm_irq_uninstall(dev);
4774         return i915_gem_idle(dev);
4775 }
4776
4777 void
4778 i915_gem_lastclose(struct drm_device *dev)
4779 {
4780         int ret;
4781
4782         if (drm_core_check_feature(dev, DRIVER_MODESET))
4783                 return;
4784
4785         ret = i915_gem_idle(dev);
4786         if (ret)
4787                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4788 }
4789
4790 void
4791 i915_gem_load(struct drm_device *dev)
4792 {
4793         int i;
4794         drm_i915_private_t *dev_priv = dev->dev_private;
4795
4796         spin_lock_init(&dev_priv->mm.active_list_lock);
4797         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4798         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4799         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4800         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4801         INIT_LIST_HEAD(&dev_priv->mm.request_list);
4802         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4803         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4804                           i915_gem_retire_work_handler);
4805         dev_priv->mm.next_gem_seqno = 1;
4806
4807         spin_lock(&shrink_list_lock);
4808         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4809         spin_unlock(&shrink_list_lock);
4810
4811         /* Old X drivers will take 0-2 for front, back, depth buffers */
4812         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4813                 dev_priv->fence_reg_start = 3;
4814
4815         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4816                 dev_priv->num_fence_regs = 16;
4817         else
4818                 dev_priv->num_fence_regs = 8;
4819
4820         /* Initialize fence registers to zero */
4821         if (IS_I965G(dev)) {
4822                 for (i = 0; i < 16; i++)
4823                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4824         } else {
4825                 for (i = 0; i < 8; i++)
4826                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4827                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4828                         for (i = 0; i < 8; i++)
4829                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4830         }
4831         i915_gem_detect_bit_6_swizzle(dev);
4832         init_waitqueue_head(&dev_priv->pending_flip_queue);
4833 }
4834
4835 /*
4836  * Create a physically contiguous memory object for this object
4837  * e.g. for cursor + overlay regs
4838  */
4839 int i915_gem_init_phys_object(struct drm_device *dev,
4840                               int id, int size)
4841 {
4842         drm_i915_private_t *dev_priv = dev->dev_private;
4843         struct drm_i915_gem_phys_object *phys_obj;
4844         int ret;
4845
4846         if (dev_priv->mm.phys_objs[id - 1] || !size)
4847                 return 0;
4848
4849         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4850         if (!phys_obj)
4851                 return -ENOMEM;
4852
4853         phys_obj->id = id;
4854
4855         phys_obj->handle = drm_pci_alloc(dev, size, 0);
4856         if (!phys_obj->handle) {
4857                 ret = -ENOMEM;
4858                 goto kfree_obj;
4859         }
4860 #ifdef CONFIG_X86
4861         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4862 #endif
4863
4864         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4865
4866         return 0;
4867 kfree_obj:
4868         kfree(phys_obj);
4869         return ret;
4870 }
4871
4872 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4873 {
4874         drm_i915_private_t *dev_priv = dev->dev_private;
4875         struct drm_i915_gem_phys_object *phys_obj;
4876
4877         if (!dev_priv->mm.phys_objs[id - 1])
4878                 return;
4879
4880         phys_obj = dev_priv->mm.phys_objs[id - 1];
4881         if (phys_obj->cur_obj) {
4882                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4883         }
4884
4885 #ifdef CONFIG_X86
4886         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4887 #endif
4888         drm_pci_free(dev, phys_obj->handle);
4889         kfree(phys_obj);
4890         dev_priv->mm.phys_objs[id - 1] = NULL;
4891 }
4892
4893 void i915_gem_free_all_phys_object(struct drm_device *dev)
4894 {
4895         int i;
4896
4897         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4898                 i915_gem_free_phys_object(dev, i);
4899 }
4900
4901 void i915_gem_detach_phys_object(struct drm_device *dev,
4902                                  struct drm_gem_object *obj)
4903 {
4904         struct drm_i915_gem_object *obj_priv;
4905         int i;
4906         int ret;
4907         int page_count;
4908
4909         obj_priv = obj->driver_private;
4910         if (!obj_priv->phys_obj)
4911                 return;
4912
4913         ret = i915_gem_object_get_pages(obj, 0);
4914         if (ret)
4915                 goto out;
4916
4917         page_count = obj->size / PAGE_SIZE;
4918
4919         for (i = 0; i < page_count; i++) {
4920                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4921                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4922
4923                 memcpy(dst, src, PAGE_SIZE);
4924                 kunmap_atomic(dst, KM_USER0);
4925         }
4926         drm_clflush_pages(obj_priv->pages, page_count);
4927         drm_agp_chipset_flush(dev);
4928
4929         i915_gem_object_put_pages(obj);
4930 out:
4931         obj_priv->phys_obj->cur_obj = NULL;
4932         obj_priv->phys_obj = NULL;
4933 }
4934
4935 int
4936 i915_gem_attach_phys_object(struct drm_device *dev,
4937                             struct drm_gem_object *obj, int id)
4938 {
4939         drm_i915_private_t *dev_priv = dev->dev_private;
4940         struct drm_i915_gem_object *obj_priv;
4941         int ret = 0;
4942         int page_count;
4943         int i;
4944
4945         if (id > I915_MAX_PHYS_OBJECT)
4946                 return -EINVAL;
4947
4948         obj_priv = obj->driver_private;
4949
4950         if (obj_priv->phys_obj) {
4951                 if (obj_priv->phys_obj->id == id)
4952                         return 0;
4953                 i915_gem_detach_phys_object(dev, obj);
4954         }
4955
4956
4957         /* create a new object */
4958         if (!dev_priv->mm.phys_objs[id - 1]) {
4959                 ret = i915_gem_init_phys_object(dev, id,
4960                                                 obj->size);
4961                 if (ret) {
4962                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4963                         goto out;
4964                 }
4965         }
4966
4967         /* bind to the object */
4968         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4969         obj_priv->phys_obj->cur_obj = obj;
4970
4971         ret = i915_gem_object_get_pages(obj, 0);
4972         if (ret) {
4973                 DRM_ERROR("failed to get page list\n");
4974                 goto out;
4975         }
4976
4977         page_count = obj->size / PAGE_SIZE;
4978
4979         for (i = 0; i < page_count; i++) {
4980                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4981                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4982
4983                 memcpy(dst, src, PAGE_SIZE);
4984                 kunmap_atomic(src, KM_USER0);
4985         }
4986
4987         i915_gem_object_put_pages(obj);
4988
4989         return 0;
4990 out:
4991         return ret;
4992 }
4993
4994 static int
4995 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4996                      struct drm_i915_gem_pwrite *args,
4997                      struct drm_file *file_priv)
4998 {
4999         struct drm_i915_gem_object *obj_priv = obj->driver_private;
5000         void *obj_addr;
5001         int ret;
5002         char __user *user_data;
5003
5004         user_data = (char __user *) (uintptr_t) args->data_ptr;
5005         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
5006
5007         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
5008         ret = copy_from_user(obj_addr, user_data, args->size);
5009         if (ret)
5010                 return -EFAULT;
5011
5012         drm_agp_chipset_flush(dev);
5013         return 0;
5014 }
5015
5016 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
5017 {
5018         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
5019
5020         /* Clean up our request list when the client is going away, so that
5021          * later retire_requests won't dereference our soon-to-be-gone
5022          * file_priv.
5023          */
5024         mutex_lock(&dev->struct_mutex);
5025         while (!list_empty(&i915_file_priv->mm.request_list))
5026                 list_del_init(i915_file_priv->mm.request_list.next);
5027         mutex_unlock(&dev->struct_mutex);
5028 }
5029
5030 static int
5031 i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
5032 {
5033         drm_i915_private_t *dev_priv, *next_dev;
5034         struct drm_i915_gem_object *obj_priv, *next_obj;
5035         int cnt = 0;
5036         int would_deadlock = 1;
5037
5038         /* "fast-path" to count number of available objects */
5039         if (nr_to_scan == 0) {
5040                 spin_lock(&shrink_list_lock);
5041                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5042                         struct drm_device *dev = dev_priv->dev;
5043
5044                         if (mutex_trylock(&dev->struct_mutex)) {
5045                                 list_for_each_entry(obj_priv,
5046                                                     &dev_priv->mm.inactive_list,
5047                                                     list)
5048                                         cnt++;
5049                                 mutex_unlock(&dev->struct_mutex);
5050                         }
5051                 }
5052                 spin_unlock(&shrink_list_lock);
5053
5054                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5055         }
5056
5057         spin_lock(&shrink_list_lock);
5058
5059         /* first scan for clean buffers */
5060         list_for_each_entry_safe(dev_priv, next_dev,
5061                                  &shrink_list, mm.shrink_list) {
5062                 struct drm_device *dev = dev_priv->dev;
5063
5064                 if (! mutex_trylock(&dev->struct_mutex))
5065                         continue;
5066
5067                 spin_unlock(&shrink_list_lock);
5068
5069                 i915_gem_retire_requests(dev);
5070
5071                 list_for_each_entry_safe(obj_priv, next_obj,
5072                                          &dev_priv->mm.inactive_list,
5073                                          list) {
5074                         if (i915_gem_object_is_purgeable(obj_priv)) {
5075                                 i915_gem_object_unbind(obj_priv->obj);
5076                                 if (--nr_to_scan <= 0)
5077                                         break;
5078                         }
5079                 }
5080
5081                 spin_lock(&shrink_list_lock);
5082                 mutex_unlock(&dev->struct_mutex);
5083
5084                 would_deadlock = 0;
5085
5086                 if (nr_to_scan <= 0)
5087                         break;
5088         }
5089
5090         /* second pass, evict/count anything still on the inactive list */
5091         list_for_each_entry_safe(dev_priv, next_dev,
5092                                  &shrink_list, mm.shrink_list) {
5093                 struct drm_device *dev = dev_priv->dev;
5094
5095                 if (! mutex_trylock(&dev->struct_mutex))
5096                         continue;
5097
5098                 spin_unlock(&shrink_list_lock);
5099
5100                 list_for_each_entry_safe(obj_priv, next_obj,
5101                                          &dev_priv->mm.inactive_list,
5102                                          list) {
5103                         if (nr_to_scan > 0) {
5104                                 i915_gem_object_unbind(obj_priv->obj);
5105                                 nr_to_scan--;
5106                         } else
5107                                 cnt++;
5108                 }
5109
5110                 spin_lock(&shrink_list_lock);
5111                 mutex_unlock(&dev->struct_mutex);
5112
5113                 would_deadlock = 0;
5114         }
5115
5116         spin_unlock(&shrink_list_lock);
5117
5118         if (would_deadlock)
5119                 return -1;
5120         else if (cnt > 0)
5121                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5122         else
5123                 return 0;
5124 }
5125
5126 static struct shrinker shrinker = {
5127         .shrink = i915_gem_shrink,
5128         .seeks = DEFAULT_SEEKS,
5129 };
5130
5131 __init void
5132 i915_gem_shrinker_init(void)
5133 {
5134     register_shrinker(&shrinker);
5135 }
5136
5137 __exit void
5138 i915_gem_shrinker_exit(void)
5139 {
5140     unregister_shrinker(&shrinker);
5141 }