drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/swap.h>
  35 #include <linux/pci.h>
  36
  37 #define I915_GEM_GPU_DOMAINS    (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
  38
  39 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  42 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  43                                              int write);
  44 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  45                                                      uint64_t offset,
  46                                                      uint64_t size);
  47 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  48 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  49 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  50                                            unsigned alignment);
  51 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  52 static int i915_gem_evict_something(struct drm_device *dev, int min_size);
  53 static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
  54 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  55                                 struct drm_i915_gem_pwrite *args,
  56                                 struct drm_file *file_priv);
  57
  58 static LIST_HEAD(shrink_list);
  59 static DEFINE_SPINLOCK(shrink_list_lock);
  60
  61 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  62                      unsigned long end)
  63 {
  64         drm_i915_private_t *dev_priv = dev->dev_private;
  65
  66         if (start >= end ||
  67             (start & (PAGE_SIZE - 1)) != 0 ||
  68             (end & (PAGE_SIZE - 1)) != 0) {
  69                 return -EINVAL;
  70         }
  71
  72         drm_mm_init(&dev_priv->mm.gtt_space, start,
  73                     end - start);
  74
  75         dev->gtt_total = (uint32_t) (end - start);
  76
  77         return 0;
  78 }
  79
  80 int
  81 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  82                     struct drm_file *file_priv)
  83 {
  84         struct drm_i915_gem_init *args = data;
  85         int ret;
  86
  87         mutex_lock(&dev->struct_mutex);
  88         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  89         mutex_unlock(&dev->struct_mutex);
  90
  91         return ret;
  92 }
  93
  94 int
  95 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  96                             struct drm_file *file_priv)
  97 {
  98         struct drm_i915_gem_get_aperture *args = data;
  99
 100         if (!(dev->driver->driver_features & DRIVER_GEM))
 101                 return -ENODEV;
 102
 103         args->aper_size = dev->gtt_total;
 104         args->aper_available_size = (args->aper_size -
 105                                      atomic_read(&dev->pin_memory));
 106
 107         return 0;
 108 }
 109
 110
 111 /**
 112  * Creates a new mm object and returns a handle to it.
 113  */
 114 int
 115 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 116                       struct drm_file *file_priv)
 117 {
 118         struct drm_i915_gem_create *args = data;
 119         struct drm_gem_object *obj;
 120         int ret;
 121         u32 handle;
 122
 123         args->size = roundup(args->size, PAGE_SIZE);
 124
 125         /* Allocate the new object */
 126         obj = drm_gem_object_alloc(dev, args->size);
 127         if (obj == NULL)
 128                 return -ENOMEM;
 129
 130         ret = drm_gem_handle_create(file_priv, obj, &handle);
 131         drm_gem_object_handle_unreference_unlocked(obj);
 132
 133         if (ret)
 134                 return ret;
 135
 136         args->handle = handle;
 137
 138         return 0;
 139 }
 140
 141 static inline int
 142 fast_shmem_read(struct page **pages,
 143                 loff_t page_base, int page_offset,
 144                 char __user *data,
 145                 int length)
 146 {
 147         char __iomem *vaddr;
 148         int unwritten;
 149
 150         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 151         if (vaddr == NULL)
 152                 return -ENOMEM;
 153         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 154         kunmap_atomic(vaddr, KM_USER0);
 155
 156         if (unwritten)
 157                 return -EFAULT;
 158
 159         return 0;
 160 }
 161
 162 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 163 {
 164         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 165         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 166
 167         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 168                 obj_priv->tiling_mode != I915_TILING_NONE;
 169 }
 170
 171 static inline int
 172 slow_shmem_copy(struct page *dst_page,
 173                 int dst_offset,
 174                 struct page *src_page,
 175                 int src_offset,
 176                 int length)
 177 {
 178         char *dst_vaddr, *src_vaddr;
 179
 180         dst_vaddr = kmap_atomic(dst_page, KM_USER0);
 181         if (dst_vaddr == NULL)
 182                 return -ENOMEM;
 183
 184         src_vaddr = kmap_atomic(src_page, KM_USER1);
 185         if (src_vaddr == NULL) {
 186                 kunmap_atomic(dst_vaddr, KM_USER0);
 187                 return -ENOMEM;
 188         }
 189
 190         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 191
 192         kunmap_atomic(src_vaddr, KM_USER1);
 193         kunmap_atomic(dst_vaddr, KM_USER0);
 194
 195         return 0;
 196 }
 197
 198 static inline int
 199 slow_shmem_bit17_copy(struct page *gpu_page,
 200                       int gpu_offset,
 201                       struct page *cpu_page,
 202                       int cpu_offset,
 203                       int length,
 204                       int is_read)
 205 {
 206         char *gpu_vaddr, *cpu_vaddr;
 207
 208         /* Use the unswizzled path if this page isn't affected. */
 209         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 210                 if (is_read)
 211                         return slow_shmem_copy(cpu_page, cpu_offset,
 212                                                gpu_page, gpu_offset, length);
 213                 else
 214                         return slow_shmem_copy(gpu_page, gpu_offset,
 215                                                cpu_page, cpu_offset, length);
 216         }
 217
 218         gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
 219         if (gpu_vaddr == NULL)
 220                 return -ENOMEM;
 221
 222         cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
 223         if (cpu_vaddr == NULL) {
 224                 kunmap_atomic(gpu_vaddr, KM_USER0);
 225                 return -ENOMEM;
 226         }
 227
 228         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 229          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 230          */
 231         while (length > 0) {
 232                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 233                 int this_length = min(cacheline_end - gpu_offset, length);
 234                 int swizzled_gpu_offset = gpu_offset ^ 64;
 235
 236                 if (is_read) {
 237                         memcpy(cpu_vaddr + cpu_offset,
 238                                gpu_vaddr + swizzled_gpu_offset,
 239                                this_length);
 240                 } else {
 241                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 242                                cpu_vaddr + cpu_offset,
 243                                this_length);
 244                 }
 245                 cpu_offset += this_length;
 246                 gpu_offset += this_length;
 247                 length -= this_length;
 248         }
 249
 250         kunmap_atomic(cpu_vaddr, KM_USER1);
 251         kunmap_atomic(gpu_vaddr, KM_USER0);
 252
 253         return 0;
 254 }
 255
 256 /**
 257  * This is the fast shmem pread path, which attempts to copy_from_user directly
 258  * from the backing pages of the object to the user's address space.  On a
 259  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 260  */
 261 static int
 262 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 263                           struct drm_i915_gem_pread *args,
 264                           struct drm_file *file_priv)
 265 {
 266         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 267         ssize_t remain;
 268         loff_t offset, page_base;
 269         char __user *user_data;
 270         int page_offset, page_length;
 271         int ret;
 272
 273         user_data = (char __user *) (uintptr_t) args->data_ptr;
 274         remain = args->size;
 275
 276         mutex_lock(&dev->struct_mutex);
 277
 278         ret = i915_gem_object_get_pages(obj, 0);
 279         if (ret != 0)
 280                 goto fail_unlock;
 281
 282         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 283                                                         args->size);
 284         if (ret != 0)
 285                 goto fail_put_pages;
 286
 287         obj_priv = obj->driver_private;
 288         offset = args->offset;
 289
 290         while (remain > 0) {
 291                 /* Operation in this page
 292                  *
 293                  * page_base = page offset within aperture
 294                  * page_offset = offset within page
 295                  * page_length = bytes to copy for this page
 296                  */
 297                 page_base = (offset & ~(PAGE_SIZE-1));
 298                 page_offset = offset & (PAGE_SIZE-1);
 299                 page_length = remain;
 300                 if ((page_offset + remain) > PAGE_SIZE)
 301                         page_length = PAGE_SIZE - page_offset;
 302
 303                 ret = fast_shmem_read(obj_priv->pages,
 304                                       page_base, page_offset,
 305                                       user_data, page_length);
 306                 if (ret)
 307                         goto fail_put_pages;
 308
 309                 remain -= page_length;
 310                 user_data += page_length;
 311                 offset += page_length;
 312         }
 313
 314 fail_put_pages:
 315         i915_gem_object_put_pages(obj);
 316 fail_unlock:
 317         mutex_unlock(&dev->struct_mutex);
 318
 319         return ret;
 320 }
 321
 322 static int
 323 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 324 {
 325         int ret;
 326
 327         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 328
 329         /* If we've insufficient memory to map in the pages, attempt
 330          * to make some space by throwing out some old buffers.
 331          */
 332         if (ret == -ENOMEM) {
 333                 struct drm_device *dev = obj->dev;
 334
 335                 ret = i915_gem_evict_something(dev, obj->size);
 336                 if (ret)
 337                         return ret;
 338
 339                 ret = i915_gem_object_get_pages(obj, 0);
 340         }
 341
 342         return ret;
 343 }
 344
 345 /**
 346  * This is the fallback shmem pread path, which allocates temporary storage
 347  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 348  * can copy out of the object's backing pages while holding the struct mutex
 349  * and not take page faults.
 350  */
 351 static int
 352 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 353                           struct drm_i915_gem_pread *args,
 354                           struct drm_file *file_priv)
 355 {
 356         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 357         struct mm_struct *mm = current->mm;
 358         struct page **user_pages;
 359         ssize_t remain;
 360         loff_t offset, pinned_pages, i;
 361         loff_t first_data_page, last_data_page, num_pages;
 362         int shmem_page_index, shmem_page_offset;
 363         int data_page_index,  data_page_offset;
 364         int page_length;
 365         int ret;
 366         uint64_t data_ptr = args->data_ptr;
 367         int do_bit17_swizzling;
 368
 369         remain = args->size;
 370
 371         /* Pin the user pages containing the data.  We can't fault while
 372          * holding the struct mutex, yet we want to hold it while
 373          * dereferencing the user data.
 374          */
 375         first_data_page = data_ptr / PAGE_SIZE;
 376         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 377         num_pages = last_data_page - first_data_page + 1;
 378
 379         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 380         if (user_pages == NULL)
 381                 return -ENOMEM;
 382
 383         down_read(&mm->mmap_sem);
 384         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 385                                       num_pages, 1, 0, user_pages, NULL);
 386         up_read(&mm->mmap_sem);
 387         if (pinned_pages < num_pages) {
 388                 ret = -EFAULT;
 389                 goto fail_put_user_pages;
 390         }
 391
 392         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 393
 394         mutex_lock(&dev->struct_mutex);
 395
 396         ret = i915_gem_object_get_pages_or_evict(obj);
 397         if (ret)
 398                 goto fail_unlock;
 399
 400         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 401                                                         args->size);
 402         if (ret != 0)
 403                 goto fail_put_pages;
 404
 405         obj_priv = obj->driver_private;
 406         offset = args->offset;
 407
 408         while (remain > 0) {
 409                 /* Operation in this page
 410                  *
 411                  * shmem_page_index = page number within shmem file
 412                  * shmem_page_offset = offset within page in shmem file
 413                  * data_page_index = page number in get_user_pages return
 414                  * data_page_offset = offset with data_page_index page.
 415                  * page_length = bytes to copy for this page
 416                  */
 417                 shmem_page_index = offset / PAGE_SIZE;
 418                 shmem_page_offset = offset & ~PAGE_MASK;
 419                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 420                 data_page_offset = data_ptr & ~PAGE_MASK;
 421
 422                 page_length = remain;
 423                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 424                         page_length = PAGE_SIZE - shmem_page_offset;
 425                 if ((data_page_offset + page_length) > PAGE_SIZE)
 426                         page_length = PAGE_SIZE - data_page_offset;
 427
 428                 if (do_bit17_swizzling) {
 429                         ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 430                                                     shmem_page_offset,
 431                                                     user_pages[data_page_index],
 432                                                     data_page_offset,
 433                                                     page_length,
 434                                                     1);
 435                 } else {
 436                         ret = slow_shmem_copy(user_pages[data_page_index],
 437                                               data_page_offset,
 438                                               obj_priv->pages[shmem_page_index],
 439                                               shmem_page_offset,
 440                                               page_length);
 441                 }
 442                 if (ret)
 443                         goto fail_put_pages;
 444
 445                 remain -= page_length;
 446                 data_ptr += page_length;
 447                 offset += page_length;
 448         }
 449
 450 fail_put_pages:
 451         i915_gem_object_put_pages(obj);
 452 fail_unlock:
 453         mutex_unlock(&dev->struct_mutex);
 454 fail_put_user_pages:
 455         for (i = 0; i < pinned_pages; i++) {
 456                 SetPageDirty(user_pages[i]);
 457                 page_cache_release(user_pages[i]);
 458         }
 459         drm_free_large(user_pages);
 460
 461         return ret;
 462 }
 463
 464 /**
 465  * Reads data from the object referenced by handle.
 466  *
 467  * On error, the contents of *data are undefined.
 468  */
 469 int
 470 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 471                      struct drm_file *file_priv)
 472 {
 473         struct drm_i915_gem_pread *args = data;
 474         struct drm_gem_object *obj;
 475         struct drm_i915_gem_object *obj_priv;
 476         int ret;
 477
 478         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 479         if (obj == NULL)
 480                 return -EBADF;
 481         obj_priv = obj->driver_private;
 482
 483         /* Bounds check source.
 484          *
 485          * XXX: This could use review for overflow issues...
 486          */
 487         if (args->offset > obj->size || args->size > obj->size ||
 488             args->offset + args->size > obj->size) {
 489                 drm_gem_object_unreference_unlocked(obj);
 490                 return -EINVAL;
 491         }
 492
 493         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 494                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 495         } else {
 496                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 497                 if (ret != 0)
 498                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 499                                                         file_priv);
 500         }
 501
 502         drm_gem_object_unreference_unlocked(obj);
 503
 504         return ret;
 505 }
 506
 507 /* This is the fast write path which cannot handle
 508  * page faults in the source data
 509  */
 510
 511 static inline int
 512 fast_user_write(struct io_mapping *mapping,
 513                 loff_t page_base, int page_offset,
 514                 char __user *user_data,
 515                 int length)
 516 {
 517         char *vaddr_atomic;
 518         unsigned long unwritten;
 519
 520         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 521         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 522                                                       user_data, length);
 523         io_mapping_unmap_atomic(vaddr_atomic);
 524         if (unwritten)
 525                 return -EFAULT;
 526         return 0;
 527 }
 528
 529 /* Here's the write path which can sleep for
 530  * page faults
 531  */
 532
 533 static inline int
 534 slow_kernel_write(struct io_mapping *mapping,
 535                   loff_t gtt_base, int gtt_offset,
 536                   struct page *user_page, int user_offset,
 537                   int length)
 538 {
 539         char *src_vaddr, *dst_vaddr;
 540         unsigned long unwritten;
 541
 542         dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
 543         src_vaddr = kmap_atomic(user_page, KM_USER1);
 544         unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
 545                                                       src_vaddr + user_offset,
 546                                                       length);
 547         kunmap_atomic(src_vaddr, KM_USER1);
 548         io_mapping_unmap_atomic(dst_vaddr);
 549         if (unwritten)
 550                 return -EFAULT;
 551         return 0;
 552 }
 553
 554 static inline int
 555 fast_shmem_write(struct page **pages,
 556                  loff_t page_base, int page_offset,
 557                  char __user *data,
 558                  int length)
 559 {
 560         char __iomem *vaddr;
 561         unsigned long unwritten;
 562
 563         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 564         if (vaddr == NULL)
 565                 return -ENOMEM;
 566         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 567         kunmap_atomic(vaddr, KM_USER0);
 568
 569         if (unwritten)
 570                 return -EFAULT;
 571         return 0;
 572 }
 573
 574 /**
 575  * This is the fast pwrite path, where we copy the data directly from the
 576  * user into the GTT, uncached.
 577  */
 578 static int
 579 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 580                          struct drm_i915_gem_pwrite *args,
 581                          struct drm_file *file_priv)
 582 {
 583         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 584         drm_i915_private_t *dev_priv = dev->dev_private;
 585         ssize_t remain;
 586         loff_t offset, page_base;
 587         char __user *user_data;
 588         int page_offset, page_length;
 589         int ret;
 590
 591         user_data = (char __user *) (uintptr_t) args->data_ptr;
 592         remain = args->size;
 593         if (!access_ok(VERIFY_READ, user_data, remain))
 594                 return -EFAULT;
 595
 596
 597         mutex_lock(&dev->struct_mutex);
 598         ret = i915_gem_object_pin(obj, 0);
 599         if (ret) {
 600                 mutex_unlock(&dev->struct_mutex);
 601                 return ret;
 602         }
 603         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 604         if (ret)
 605                 goto fail;
 606
 607         obj_priv = obj->driver_private;
 608         offset = obj_priv->gtt_offset + args->offset;
 609
 610         while (remain > 0) {
 611                 /* Operation in this page
 612                  *
 613                  * page_base = page offset within aperture
 614                  * page_offset = offset within page
 615                  * page_length = bytes to copy for this page
 616                  */
 617                 page_base = (offset & ~(PAGE_SIZE-1));
 618                 page_offset = offset & (PAGE_SIZE-1);
 619                 page_length = remain;
 620                 if ((page_offset + remain) > PAGE_SIZE)
 621                         page_length = PAGE_SIZE - page_offset;
 622
 623                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 624                                        page_offset, user_data, page_length);
 625
 626                 /* If we get a fault while copying data, then (presumably) our
 627                  * source page isn't available.  Return the error and we'll
 628                  * retry in the slow path.
 629                  */
 630                 if (ret)
 631                         goto fail;
 632
 633                 remain -= page_length;
 634                 user_data += page_length;
 635                 offset += page_length;
 636         }
 637
 638 fail:
 639         i915_gem_object_unpin(obj);
 640         mutex_unlock(&dev->struct_mutex);
 641
 642         return ret;
 643 }
 644
 645 /**
 646  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 647  * the memory and maps it using kmap_atomic for copying.
 648  *
 649  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 650  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 651  */
 652 static int
 653 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 654                          struct drm_i915_gem_pwrite *args,
 655                          struct drm_file *file_priv)
 656 {
 657         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 658         drm_i915_private_t *dev_priv = dev->dev_private;
 659         ssize_t remain;
 660         loff_t gtt_page_base, offset;
 661         loff_t first_data_page, last_data_page, num_pages;
 662         loff_t pinned_pages, i;
 663         struct page **user_pages;
 664         struct mm_struct *mm = current->mm;
 665         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 666         int ret;
 667         uint64_t data_ptr = args->data_ptr;
 668
 669         remain = args->size;
 670
 671         /* Pin the user pages containing the data.  We can't fault while
 672          * holding the struct mutex, and all of the pwrite implementations
 673          * want to hold it while dereferencing the user data.
 674          */
 675         first_data_page = data_ptr / PAGE_SIZE;
 676         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 677         num_pages = last_data_page - first_data_page + 1;
 678
 679         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 680         if (user_pages == NULL)
 681                 return -ENOMEM;
 682
 683         down_read(&mm->mmap_sem);
 684         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 685                                       num_pages, 0, 0, user_pages, NULL);
 686         up_read(&mm->mmap_sem);
 687         if (pinned_pages < num_pages) {
 688                 ret = -EFAULT;
 689                 goto out_unpin_pages;
 690         }
 691
 692         mutex_lock(&dev->struct_mutex);
 693         ret = i915_gem_object_pin(obj, 0);
 694         if (ret)
 695                 goto out_unlock;
 696
 697         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 698         if (ret)
 699                 goto out_unpin_object;
 700
 701         obj_priv = obj->driver_private;
 702         offset = obj_priv->gtt_offset + args->offset;
 703
 704         while (remain > 0) {
 705                 /* Operation in this page
 706                  *
 707                  * gtt_page_base = page offset within aperture
 708                  * gtt_page_offset = offset within page in aperture
 709                  * data_page_index = page number in get_user_pages return
 710                  * data_page_offset = offset with data_page_index page.
 711                  * page_length = bytes to copy for this page
 712                  */
 713                 gtt_page_base = offset & PAGE_MASK;
 714                 gtt_page_offset = offset & ~PAGE_MASK;
 715                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 716                 data_page_offset = data_ptr & ~PAGE_MASK;
 717
 718                 page_length = remain;
 719                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 720                         page_length = PAGE_SIZE - gtt_page_offset;
 721                 if ((data_page_offset + page_length) > PAGE_SIZE)
 722                         page_length = PAGE_SIZE - data_page_offset;
 723
 724                 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
 725                                         gtt_page_base, gtt_page_offset,
 726                                         user_pages[data_page_index],
 727                                         data_page_offset,
 728                                         page_length);
 729
 730                 /* If we get a fault while copying data, then (presumably) our
 731                  * source page isn't available.  Return the error and we'll
 732                  * retry in the slow path.
 733                  */
 734                 if (ret)
 735                         goto out_unpin_object;
 736
 737                 remain -= page_length;
 738                 offset += page_length;
 739                 data_ptr += page_length;
 740         }
 741
 742 out_unpin_object:
 743         i915_gem_object_unpin(obj);
 744 out_unlock:
 745         mutex_unlock(&dev->struct_mutex);
 746 out_unpin_pages:
 747         for (i = 0; i < pinned_pages; i++)
 748                 page_cache_release(user_pages[i]);
 749         drm_free_large(user_pages);
 750
 751         return ret;
 752 }
 753
 754 /**
 755  * This is the fast shmem pwrite path, which attempts to directly
 756  * copy_from_user into the kmapped pages backing the object.
 757  */
 758 static int
 759 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 760                            struct drm_i915_gem_pwrite *args,
 761                            struct drm_file *file_priv)
 762 {
 763         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 764         ssize_t remain;
 765         loff_t offset, page_base;
 766         char __user *user_data;
 767         int page_offset, page_length;
 768         int ret;
 769
 770         user_data = (char __user *) (uintptr_t) args->data_ptr;
 771         remain = args->size;
 772
 773         mutex_lock(&dev->struct_mutex);
 774
 775         ret = i915_gem_object_get_pages(obj, 0);
 776         if (ret != 0)
 777                 goto fail_unlock;
 778
 779         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 780         if (ret != 0)
 781                 goto fail_put_pages;
 782
 783         obj_priv = obj->driver_private;
 784         offset = args->offset;
 785         obj_priv->dirty = 1;
 786
 787         while (remain > 0) {
 788                 /* Operation in this page
 789                  *
 790                  * page_base = page offset within aperture
 791                  * page_offset = offset within page
 792                  * page_length = bytes to copy for this page
 793                  */
 794                 page_base = (offset & ~(PAGE_SIZE-1));
 795                 page_offset = offset & (PAGE_SIZE-1);
 796                 page_length = remain;
 797                 if ((page_offset + remain) > PAGE_SIZE)
 798                         page_length = PAGE_SIZE - page_offset;
 799
 800                 ret = fast_shmem_write(obj_priv->pages,
 801                                        page_base, page_offset,
 802                                        user_data, page_length);
 803                 if (ret)
 804                         goto fail_put_pages;
 805
 806                 remain -= page_length;
 807                 user_data += page_length;
 808                 offset += page_length;
 809         }
 810
 811 fail_put_pages:
 812         i915_gem_object_put_pages(obj);
 813 fail_unlock:
 814         mutex_unlock(&dev->struct_mutex);
 815
 816         return ret;
 817 }
 818
 819 /**
 820  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 821  * the memory and maps it using kmap_atomic for copying.
 822  *
 823  * This avoids taking mmap_sem for faulting on the user's address while the
 824  * struct_mutex is held.
 825  */
 826 static int
 827 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 828                            struct drm_i915_gem_pwrite *args,
 829                            struct drm_file *file_priv)
 830 {
 831         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 832         struct mm_struct *mm = current->mm;
 833         struct page **user_pages;
 834         ssize_t remain;
 835         loff_t offset, pinned_pages, i;
 836         loff_t first_data_page, last_data_page, num_pages;
 837         int shmem_page_index, shmem_page_offset;
 838         int data_page_index,  data_page_offset;
 839         int page_length;
 840         int ret;
 841         uint64_t data_ptr = args->data_ptr;
 842         int do_bit17_swizzling;
 843
 844         remain = args->size;
 845
 846         /* Pin the user pages containing the data.  We can't fault while
 847          * holding the struct mutex, and all of the pwrite implementations
 848          * want to hold it while dereferencing the user data.
 849          */
 850         first_data_page = data_ptr / PAGE_SIZE;
 851         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 852         num_pages = last_data_page - first_data_page + 1;
 853
 854         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 855         if (user_pages == NULL)
 856                 return -ENOMEM;
 857
 858         down_read(&mm->mmap_sem);
 859         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 860                                       num_pages, 0, 0, user_pages, NULL);
 861         up_read(&mm->mmap_sem);
 862         if (pinned_pages < num_pages) {
 863                 ret = -EFAULT;
 864                 goto fail_put_user_pages;
 865         }
 866
 867         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 868
 869         mutex_lock(&dev->struct_mutex);
 870
 871         ret = i915_gem_object_get_pages_or_evict(obj);
 872         if (ret)
 873                 goto fail_unlock;
 874
 875         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 876         if (ret != 0)
 877                 goto fail_put_pages;
 878
 879         obj_priv = obj->driver_private;
 880         offset = args->offset;
 881         obj_priv->dirty = 1;
 882
 883         while (remain > 0) {
 884                 /* Operation in this page
 885                  *
 886                  * shmem_page_index = page number within shmem file
 887                  * shmem_page_offset = offset within page in shmem file
 888                  * data_page_index = page number in get_user_pages return
 889                  * data_page_offset = offset with data_page_index page.
 890                  * page_length = bytes to copy for this page
 891                  */
 892                 shmem_page_index = offset / PAGE_SIZE;
 893                 shmem_page_offset = offset & ~PAGE_MASK;
 894                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 895                 data_page_offset = data_ptr & ~PAGE_MASK;
 896
 897                 page_length = remain;
 898                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 899                         page_length = PAGE_SIZE - shmem_page_offset;
 900                 if ((data_page_offset + page_length) > PAGE_SIZE)
 901                         page_length = PAGE_SIZE - data_page_offset;
 902
 903                 if (do_bit17_swizzling) {
 904                         ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 905                                                     shmem_page_offset,
 906                                                     user_pages[data_page_index],
 907                                                     data_page_offset,
 908                                                     page_length,
 909                                                     0);
 910                 } else {
 911                         ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
 912                                               shmem_page_offset,
 913                                               user_pages[data_page_index],
 914                                               data_page_offset,
 915                                               page_length);
 916                 }
 917                 if (ret)
 918                         goto fail_put_pages;
 919
 920                 remain -= page_length;
 921                 data_ptr += page_length;
 922                 offset += page_length;
 923         }
 924
 925 fail_put_pages:
 926         i915_gem_object_put_pages(obj);
 927 fail_unlock:
 928         mutex_unlock(&dev->struct_mutex);
 929 fail_put_user_pages:
 930         for (i = 0; i < pinned_pages; i++)
 931                 page_cache_release(user_pages[i]);
 932         drm_free_large(user_pages);
 933
 934         return ret;
 935 }
 936
 937 /**
 938  * Writes data to the object referenced by handle.
 939  *
 940  * On error, the contents of the buffer that were to be modified are undefined.
 941  */
 942 int
 943 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 944                       struct drm_file *file_priv)
 945 {
 946         struct drm_i915_gem_pwrite *args = data;
 947         struct drm_gem_object *obj;
 948         struct drm_i915_gem_object *obj_priv;
 949         int ret = 0;
 950
 951         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 952         if (obj == NULL)
 953                 return -EBADF;
 954         obj_priv = obj->driver_private;
 955
 956         /* Bounds check destination.
 957          *
 958          * XXX: This could use review for overflow issues...
 959          */
 960         if (args->offset > obj->size || args->size > obj->size ||
 961             args->offset + args->size > obj->size) {
 962                 drm_gem_object_unreference_unlocked(obj);
 963                 return -EINVAL;
 964         }
 965
 966         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 967          * it would end up going through the fenced access, and we'll get
 968          * different detiling behavior between reading and writing.
 969          * pread/pwrite currently are reading and writing from the CPU
 970          * perspective, requiring manual detiling by the client.
 971          */
 972         if (obj_priv->phys_obj)
 973                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 974         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 975                  dev->gtt_total != 0) {
 976                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 977                 if (ret == -EFAULT) {
 978                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 979                                                        file_priv);
 980                 }
 981         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 982                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 983         } else {
 984                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 985                 if (ret == -EFAULT) {
 986                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 987                                                          file_priv);
 988                 }
 989         }
 990
 991 #if WATCH_PWRITE
 992         if (ret)
 993                 DRM_INFO("pwrite failed %d\n", ret);
 994 #endif
 995
 996         drm_gem_object_unreference_unlocked(obj);
 997
 998         return ret;
 999 }
1000
1001 /**
1002  * Called when user space prepares to use an object with the CPU, either
1003  * through the mmap ioctl's mapping or a GTT mapping.
1004  */
1005 int
1006 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1007                           struct drm_file *file_priv)
1008 {
1009         struct drm_i915_private *dev_priv = dev->dev_private;
1010         struct drm_i915_gem_set_domain *args = data;
1011         struct drm_gem_object *obj;
1012         struct drm_i915_gem_object *obj_priv;
1013         uint32_t read_domains = args->read_domains;
1014         uint32_t write_domain = args->write_domain;
1015         int ret;
1016
1017         if (!(dev->driver->driver_features & DRIVER_GEM))
1018                 return -ENODEV;
1019
1020         /* Only handle setting domains to types used by the CPU. */
1021         if (write_domain & I915_GEM_GPU_DOMAINS)
1022                 return -EINVAL;
1023
1024         if (read_domains & I915_GEM_GPU_DOMAINS)
1025                 return -EINVAL;
1026
1027         /* Having something in the write domain implies it's in the read
1028          * domain, and only that read domain.  Enforce that in the request.
1029          */
1030         if (write_domain != 0 && read_domains != write_domain)
1031                 return -EINVAL;
1032
1033         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1034         if (obj == NULL)
1035                 return -EBADF;
1036         obj_priv = obj->driver_private;
1037
1038         mutex_lock(&dev->struct_mutex);
1039
1040         intel_mark_busy(dev, obj);
1041
1042 #if WATCH_BUF
1043         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1044                  obj, obj->size, read_domains, write_domain);
1045 #endif
1046         if (read_domains & I915_GEM_DOMAIN_GTT) {
1047                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1048
1049                 /* Update the LRU on the fence for the CPU access that's
1050                  * about to occur.
1051                  */
1052                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1053                         list_move_tail(&obj_priv->fence_list,
1054                                        &dev_priv->mm.fence_list);
1055                 }
1056
1057                 /* Silently promote "you're not bound, there was nothing to do"
1058                  * to success, since the client was just asking us to
1059                  * make sure everything was done.
1060                  */
1061                 if (ret == -EINVAL)
1062                         ret = 0;
1063         } else {
1064                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1065         }
1066
1067         drm_gem_object_unreference(obj);
1068         mutex_unlock(&dev->struct_mutex);
1069         return ret;
1070 }
1071
1072 /**
1073  * Called when user space has done writes to this buffer
1074  */
1075 int
1076 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1077                       struct drm_file *file_priv)
1078 {
1079         struct drm_i915_gem_sw_finish *args = data;
1080         struct drm_gem_object *obj;
1081         struct drm_i915_gem_object *obj_priv;
1082         int ret = 0;
1083
1084         if (!(dev->driver->driver_features & DRIVER_GEM))
1085                 return -ENODEV;
1086
1087         mutex_lock(&dev->struct_mutex);
1088         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1089         if (obj == NULL) {
1090                 mutex_unlock(&dev->struct_mutex);
1091                 return -EBADF;
1092         }
1093
1094 #if WATCH_BUF
1095         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1096                  __func__, args->handle, obj, obj->size);
1097 #endif
1098         obj_priv = obj->driver_private;
1099
1100         /* Pinned buffers may be scanout, so flush the cache */
1101         if (obj_priv->pin_count)
1102                 i915_gem_object_flush_cpu_write_domain(obj);
1103
1104         drm_gem_object_unreference(obj);
1105         mutex_unlock(&dev->struct_mutex);
1106         return ret;
1107 }
1108
1109 /**
1110  * Maps the contents of an object, returning the address it is mapped
1111  * into.
1112  *
1113  * While the mapping holds a reference on the contents of the object, it doesn't
1114  * imply a ref on the object itself.
1115  */
1116 int
1117 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1118                    struct drm_file *file_priv)
1119 {
1120         struct drm_i915_gem_mmap *args = data;
1121         struct drm_gem_object *obj;
1122         loff_t offset;
1123         unsigned long addr;
1124
1125         if (!(dev->driver->driver_features & DRIVER_GEM))
1126                 return -ENODEV;
1127
1128         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1129         if (obj == NULL)
1130                 return -EBADF;
1131
1132         offset = args->offset;
1133
1134         down_write(&current->mm->mmap_sem);
1135         addr = do_mmap(obj->filp, 0, args->size,
1136                        PROT_READ | PROT_WRITE, MAP_SHARED,
1137                        args->offset);
1138         up_write(&current->mm->mmap_sem);
1139         drm_gem_object_unreference_unlocked(obj);
1140         if (IS_ERR((void *)addr))
1141                 return addr;
1142
1143         args->addr_ptr = (uint64_t) addr;
1144
1145         return 0;
1146 }
1147
1148 /**
1149  * i915_gem_fault - fault a page into the GTT
1150  * vma: VMA in question
1151  * vmf: fault info
1152  *
1153  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1154  * from userspace.  The fault handler takes care of binding the object to
1155  * the GTT (if needed), allocating and programming a fence register (again,
1156  * only if needed based on whether the old reg is still valid or the object
1157  * is tiled) and inserting a new PTE into the faulting process.
1158  *
1159  * Note that the faulting process may involve evicting existing objects
1160  * from the GTT and/or fence registers to make room.  So performance may
1161  * suffer if the GTT working set is large or there are few fence registers
1162  * left.
1163  */
1164 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1165 {
1166         struct drm_gem_object *obj = vma->vm_private_data;
1167         struct drm_device *dev = obj->dev;
1168         struct drm_i915_private *dev_priv = dev->dev_private;
1169         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1170         pgoff_t page_offset;
1171         unsigned long pfn;
1172         int ret = 0;
1173         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1174
1175         /* We don't use vmf->pgoff since that has the fake offset */
1176         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1177                 PAGE_SHIFT;
1178
1179         /* Now bind it into the GTT if needed */
1180         mutex_lock(&dev->struct_mutex);
1181         if (!obj_priv->gtt_space) {
1182                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1183                 if (ret)
1184                         goto unlock;
1185
1186                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1187
1188                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1189                 if (ret)
1190                         goto unlock;
1191         }
1192
1193         /* Need a new fence register? */
1194         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1195                 ret = i915_gem_object_get_fence_reg(obj);
1196                 if (ret)
1197                         goto unlock;
1198         }
1199
1200         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1201                 page_offset;
1202
1203         /* Finally, remap it using the new GTT offset */
1204         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1205 unlock:
1206         mutex_unlock(&dev->struct_mutex);
1207
1208         switch (ret) {
1209         case 0:
1210         case -ERESTARTSYS:
1211                 return VM_FAULT_NOPAGE;
1212         case -ENOMEM:
1213         case -EAGAIN:
1214                 return VM_FAULT_OOM;
1215         default:
1216                 return VM_FAULT_SIGBUS;
1217         }
1218 }
1219
1220 /**
1221  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1222  * @obj: obj in question
1223  *
1224  * GEM memory mapping works by handing back to userspace a fake mmap offset
1225  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1226  * up the object based on the offset and sets up the various memory mapping
1227  * structures.
1228  *
1229  * This routine allocates and attaches a fake offset for @obj.
1230  */
1231 static int
1232 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1233 {
1234         struct drm_device *dev = obj->dev;
1235         struct drm_gem_mm *mm = dev->mm_private;
1236         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1237         struct drm_map_list *list;
1238         struct drm_local_map *map;
1239         int ret = 0;
1240
1241         /* Set the object up for mmap'ing */
1242         list = &obj->map_list;
1243         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1244         if (!list->map)
1245                 return -ENOMEM;
1246
1247         map = list->map;
1248         map->type = _DRM_GEM;
1249         map->size = obj->size;
1250         map->handle = obj;
1251
1252         /* Get a DRM GEM mmap offset allocated... */
1253         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1254                                                     obj->size / PAGE_SIZE, 0, 0);
1255         if (!list->file_offset_node) {
1256                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1257                 ret = -ENOMEM;
1258                 goto out_free_list;
1259         }
1260
1261         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1262                                                   obj->size / PAGE_SIZE, 0);
1263         if (!list->file_offset_node) {
1264                 ret = -ENOMEM;
1265                 goto out_free_list;
1266         }
1267
1268         list->hash.key = list->file_offset_node->start;
1269         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1270                 DRM_ERROR("failed to add to map hash\n");
1271                 ret = -ENOMEM;
1272                 goto out_free_mm;
1273         }
1274
1275         /* By now we should be all set, any drm_mmap request on the offset
1276          * below will get to our mmap & fault handler */
1277         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1278
1279         return 0;
1280
1281 out_free_mm:
1282         drm_mm_put_block(list->file_offset_node);
1283 out_free_list:
1284         kfree(list->map);
1285
1286         return ret;
1287 }
1288
1289 /**
1290  * i915_gem_release_mmap - remove physical page mappings
1291  * @obj: obj in question
1292  *
1293  * Preserve the reservation of the mmapping with the DRM core code, but
1294  * relinquish ownership of the pages back to the system.
1295  *
1296  * It is vital that we remove the page mapping if we have mapped a tiled
1297  * object through the GTT and then lose the fence register due to
1298  * resource pressure. Similarly if the object has been moved out of the
1299  * aperture, than pages mapped into userspace must be revoked. Removing the
1300  * mapping will then trigger a page fault on the next user access, allowing
1301  * fixup by i915_gem_fault().
1302  */
1303 void
1304 i915_gem_release_mmap(struct drm_gem_object *obj)
1305 {
1306         struct drm_device *dev = obj->dev;
1307         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1308
1309         if (dev->dev_mapping)
1310                 unmap_mapping_range(dev->dev_mapping,
1311                                     obj_priv->mmap_offset, obj->size, 1);
1312 }
1313
1314 static void
1315 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1316 {
1317         struct drm_device *dev = obj->dev;
1318         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1319         struct drm_gem_mm *mm = dev->mm_private;
1320         struct drm_map_list *list;
1321
1322         list = &obj->map_list;
1323         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1324
1325         if (list->file_offset_node) {
1326                 drm_mm_put_block(list->file_offset_node);
1327                 list->file_offset_node = NULL;
1328         }
1329
1330         if (list->map) {
1331                 kfree(list->map);
1332                 list->map = NULL;
1333         }
1334
1335         obj_priv->mmap_offset = 0;
1336 }
1337
1338 /**
1339  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1340  * @obj: object to check
1341  *
1342  * Return the required GTT alignment for an object, taking into account
1343  * potential fence register mapping if needed.
1344  */
1345 static uint32_t
1346 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1347 {
1348         struct drm_device *dev = obj->dev;
1349         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1350         int start, i;
1351
1352         /*
1353          * Minimum alignment is 4k (GTT page size), but might be greater
1354          * if a fence register is needed for the object.
1355          */
1356         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1357                 return 4096;
1358
1359         /*
1360          * Previous chips need to be aligned to the size of the smallest
1361          * fence register that can contain the object.
1362          */
1363         if (IS_I9XX(dev))
1364                 start = 1024*1024;
1365         else
1366                 start = 512*1024;
1367
1368         for (i = start; i < obj->size; i <<= 1)
1369                 ;
1370
1371         return i;
1372 }
1373
1374 /**
1375  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1376  * @dev: DRM device
1377  * @data: GTT mapping ioctl data
1378  * @file_priv: GEM object info
1379  *
1380  * Simply returns the fake offset to userspace so it can mmap it.
1381  * The mmap call will end up in drm_gem_mmap(), which will set things
1382  * up so we can get faults in the handler above.
1383  *
1384  * The fault handler will take care of binding the object into the GTT
1385  * (since it may have been evicted to make room for something), allocating
1386  * a fence register, and mapping the appropriate aperture address into
1387  * userspace.
1388  */
1389 int
1390 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1391                         struct drm_file *file_priv)
1392 {
1393         struct drm_i915_gem_mmap_gtt *args = data;
1394         struct drm_i915_private *dev_priv = dev->dev_private;
1395         struct drm_gem_object *obj;
1396         struct drm_i915_gem_object *obj_priv;
1397         int ret;
1398
1399         if (!(dev->driver->driver_features & DRIVER_GEM))
1400                 return -ENODEV;
1401
1402         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1403         if (obj == NULL)
1404                 return -EBADF;
1405
1406         mutex_lock(&dev->struct_mutex);
1407
1408         obj_priv = obj->driver_private;
1409
1410         if (obj_priv->madv != I915_MADV_WILLNEED) {
1411                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1412                 drm_gem_object_unreference(obj);
1413                 mutex_unlock(&dev->struct_mutex);
1414                 return -EINVAL;
1415         }
1416
1417
1418         if (!obj_priv->mmap_offset) {
1419                 ret = i915_gem_create_mmap_offset(obj);
1420                 if (ret) {
1421                         drm_gem_object_unreference(obj);
1422                         mutex_unlock(&dev->struct_mutex);
1423                         return ret;
1424                 }
1425         }
1426
1427         args->offset = obj_priv->mmap_offset;
1428
1429         /*
1430          * Pull it into the GTT so that we have a page list (makes the
1431          * initial fault faster and any subsequent flushing possible).
1432          */
1433         if (!obj_priv->agp_mem) {
1434                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1435                 if (ret) {
1436                         drm_gem_object_unreference(obj);
1437                         mutex_unlock(&dev->struct_mutex);
1438                         return ret;
1439                 }
1440                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1441         }
1442
1443         drm_gem_object_unreference(obj);
1444         mutex_unlock(&dev->struct_mutex);
1445
1446         return 0;
1447 }
1448
1449 void
1450 i915_gem_object_put_pages(struct drm_gem_object *obj)
1451 {
1452         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1453         int page_count = obj->size / PAGE_SIZE;
1454         int i;
1455
1456         BUG_ON(obj_priv->pages_refcount == 0);
1457         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1458
1459         if (--obj_priv->pages_refcount != 0)
1460                 return;
1461
1462         if (obj_priv->tiling_mode != I915_TILING_NONE)
1463                 i915_gem_object_save_bit_17_swizzle(obj);
1464
1465         if (obj_priv->madv == I915_MADV_DONTNEED)
1466                 obj_priv->dirty = 0;
1467
1468         for (i = 0; i < page_count; i++) {
1469                 if (obj_priv->pages[i] == NULL)
1470                         break;
1471
1472                 if (obj_priv->dirty)
1473                         set_page_dirty(obj_priv->pages[i]);
1474
1475                 if (obj_priv->madv == I915_MADV_WILLNEED)
1476                         mark_page_accessed(obj_priv->pages[i]);
1477
1478                 page_cache_release(obj_priv->pages[i]);
1479         }
1480         obj_priv->dirty = 0;
1481
1482         drm_free_large(obj_priv->pages);
1483         obj_priv->pages = NULL;
1484 }
1485
1486 static void
1487 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
1488 {
1489         struct drm_device *dev = obj->dev;
1490         drm_i915_private_t *dev_priv = dev->dev_private;
1491         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1492
1493         /* Add a reference if we're newly entering the active list. */
1494         if (!obj_priv->active) {
1495                 drm_gem_object_reference(obj);
1496                 obj_priv->active = 1;
1497         }
1498         /* Move from whatever list we were on to the tail of execution. */
1499         spin_lock(&dev_priv->mm.active_list_lock);
1500         list_move_tail(&obj_priv->list,
1501                        &dev_priv->mm.active_list);
1502         spin_unlock(&dev_priv->mm.active_list_lock);
1503         obj_priv->last_rendering_seqno = seqno;
1504 }
1505
1506 static void
1507 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1508 {
1509         struct drm_device *dev = obj->dev;
1510         drm_i915_private_t *dev_priv = dev->dev_private;
1511         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1512
1513         BUG_ON(!obj_priv->active);
1514         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1515         obj_priv->last_rendering_seqno = 0;
1516 }
1517
1518 /* Immediately discard the backing storage */
1519 static void
1520 i915_gem_object_truncate(struct drm_gem_object *obj)
1521 {
1522         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1523         struct inode *inode;
1524
1525         inode = obj->filp->f_path.dentry->d_inode;
1526         if (inode->i_op->truncate)
1527                 inode->i_op->truncate (inode);
1528
1529         obj_priv->madv = __I915_MADV_PURGED;
1530 }
1531
1532 static inline int
1533 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1534 {
1535         return obj_priv->madv == I915_MADV_DONTNEED;
1536 }
1537
1538 static void
1539 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1540 {
1541         struct drm_device *dev = obj->dev;
1542         drm_i915_private_t *dev_priv = dev->dev_private;
1543         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1544
1545         i915_verify_inactive(dev, __FILE__, __LINE__);
1546         if (obj_priv->pin_count != 0)
1547                 list_del_init(&obj_priv->list);
1548         else
1549                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1550
1551         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1552
1553         obj_priv->last_rendering_seqno = 0;
1554         if (obj_priv->active) {
1555                 obj_priv->active = 0;
1556                 drm_gem_object_unreference(obj);
1557         }
1558         i915_verify_inactive(dev, __FILE__, __LINE__);
1559 }
1560
1561 static void
1562 i915_gem_process_flushing_list(struct drm_device *dev,
1563                                uint32_t flush_domains, uint32_t seqno)
1564 {
1565         drm_i915_private_t *dev_priv = dev->dev_private;
1566         struct drm_i915_gem_object *obj_priv, *next;
1567
1568         list_for_each_entry_safe(obj_priv, next,
1569                                  &dev_priv->mm.gpu_write_list,
1570                                  gpu_write_list) {
1571                 struct drm_gem_object *obj = obj_priv->obj;
1572
1573                 if ((obj->write_domain & flush_domains) ==
1574                     obj->write_domain) {
1575                         uint32_t old_write_domain = obj->write_domain;
1576
1577                         obj->write_domain = 0;
1578                         list_del_init(&obj_priv->gpu_write_list);
1579                         i915_gem_object_move_to_active(obj, seqno);
1580
1581                         /* update the fence lru list */
1582                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1583                                 list_move_tail(&obj_priv->fence_list,
1584                                                 &dev_priv->mm.fence_list);
1585
1586                         trace_i915_gem_object_change_domain(obj,
1587                                                             obj->read_domains,
1588                                                             old_write_domain);
1589                 }
1590         }
1591 }
1592
1593 /**
1594  * Creates a new sequence number, emitting a write of it to the status page
1595  * plus an interrupt, which will trigger i915_user_interrupt_handler.
1596  *
1597  * Must be called with struct_lock held.
1598  *
1599  * Returned sequence numbers are nonzero on success.
1600  */
1601 uint32_t
1602 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1603                  uint32_t flush_domains)
1604 {
1605         drm_i915_private_t *dev_priv = dev->dev_private;
1606         struct drm_i915_file_private *i915_file_priv = NULL;
1607         struct drm_i915_gem_request *request;
1608         uint32_t seqno;
1609         int was_empty;
1610         RING_LOCALS;
1611
1612         if (file_priv != NULL)
1613                 i915_file_priv = file_priv->driver_priv;
1614
1615         request = kzalloc(sizeof(*request), GFP_KERNEL);
1616         if (request == NULL)
1617                 return 0;
1618
1619         /* Grab the seqno we're going to make this request be, and bump the
1620          * next (skipping 0 so it can be the reserved no-seqno value).
1621          */
1622         seqno = dev_priv->mm.next_gem_seqno;
1623         dev_priv->mm.next_gem_seqno++;
1624         if (dev_priv->mm.next_gem_seqno == 0)
1625                 dev_priv->mm.next_gem_seqno++;
1626
1627         BEGIN_LP_RING(4);
1628         OUT_RING(MI_STORE_DWORD_INDEX);
1629         OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1630         OUT_RING(seqno);
1631
1632         OUT_RING(MI_USER_INTERRUPT);
1633         ADVANCE_LP_RING();
1634
1635         DRM_DEBUG_DRIVER("%d\n", seqno);
1636
1637         request->seqno = seqno;
1638         request->emitted_jiffies = jiffies;
1639         was_empty = list_empty(&dev_priv->mm.request_list);
1640         list_add_tail(&request->list, &dev_priv->mm.request_list);
1641         if (i915_file_priv) {
1642                 list_add_tail(&request->client_list,
1643                               &i915_file_priv->mm.request_list);
1644         } else {
1645                 INIT_LIST_HEAD(&request->client_list);
1646         }
1647
1648         /* Associate any objects on the flushing list matching the write
1649          * domain we're flushing with our flush.
1650          */
1651         if (flush_domains != 0)
1652                 i915_gem_process_flushing_list(dev, flush_domains, seqno);
1653
1654         if (!dev_priv->mm.suspended) {
1655                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1656                 if (was_empty)
1657                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1658         }
1659         return seqno;
1660 }
1661
1662 /**
1663  * Command execution barrier
1664  *
1665  * Ensures that all commands in the ring are finished
1666  * before signalling the CPU
1667  */
1668 static uint32_t
1669 i915_retire_commands(struct drm_device *dev)
1670 {
1671         drm_i915_private_t *dev_priv = dev->dev_private;
1672         uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1673         uint32_t flush_domains = 0;
1674         RING_LOCALS;
1675
1676         /* The sampler always gets flushed on i965 (sigh) */
1677         if (IS_I965G(dev))
1678                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1679         BEGIN_LP_RING(2);
1680         OUT_RING(cmd);
1681         OUT_RING(0); /* noop */
1682         ADVANCE_LP_RING();
1683         return flush_domains;
1684 }
1685
1686 /**
1687  * Moves buffers associated only with the given active seqno from the active
1688  * to inactive list, potentially freeing them.
1689  */
1690 static void
1691 i915_gem_retire_request(struct drm_device *dev,
1692                         struct drm_i915_gem_request *request)
1693 {
1694         drm_i915_private_t *dev_priv = dev->dev_private;
1695
1696         trace_i915_gem_request_retire(dev, request->seqno);
1697
1698         /* Move any buffers on the active list that are no longer referenced
1699          * by the ringbuffer to the flushing/inactive lists as appropriate.
1700          */
1701         spin_lock(&dev_priv->mm.active_list_lock);
1702         while (!list_empty(&dev_priv->mm.active_list)) {
1703                 struct drm_gem_object *obj;
1704                 struct drm_i915_gem_object *obj_priv;
1705
1706                 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1707                                             struct drm_i915_gem_object,
1708                                             list);
1709                 obj = obj_priv->obj;
1710
1711                 /* If the seqno being retired doesn't match the oldest in the
1712                  * list, then the oldest in the list must still be newer than
1713                  * this seqno.
1714                  */
1715                 if (obj_priv->last_rendering_seqno != request->seqno)
1716                         goto out;
1717
1718 #if WATCH_LRU
1719                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1720                          __func__, request->seqno, obj);
1721 #endif
1722
1723                 if (obj->write_domain != 0)
1724                         i915_gem_object_move_to_flushing(obj);
1725                 else {
1726                         /* Take a reference on the object so it won't be
1727                          * freed while the spinlock is held.  The list
1728                          * protection for this spinlock is safe when breaking
1729                          * the lock like this since the next thing we do
1730                          * is just get the head of the list again.
1731                          */
1732                         drm_gem_object_reference(obj);
1733                         i915_gem_object_move_to_inactive(obj);
1734                         spin_unlock(&dev_priv->mm.active_list_lock);
1735                         drm_gem_object_unreference(obj);
1736                         spin_lock(&dev_priv->mm.active_list_lock);
1737                 }
1738         }
1739 out:
1740         spin_unlock(&dev_priv->mm.active_list_lock);
1741 }
1742
1743 /**
1744  * Returns true if seq1 is later than seq2.
1745  */
1746 bool
1747 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1748 {
1749         return (int32_t)(seq1 - seq2) >= 0;
1750 }
1751
1752 uint32_t
1753 i915_get_gem_seqno(struct drm_device *dev)
1754 {
1755         drm_i915_private_t *dev_priv = dev->dev_private;
1756
1757         return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1758 }
1759
1760 /**
1761  * This function clears the request list as sequence numbers are passed.
1762  */
1763 void
1764 i915_gem_retire_requests(struct drm_device *dev)
1765 {
1766         drm_i915_private_t *dev_priv = dev->dev_private;
1767         uint32_t seqno;
1768
1769         if (!dev_priv->hw_status_page || list_empty(&dev_priv->mm.request_list))
1770                 return;
1771
1772         seqno = i915_get_gem_seqno(dev);
1773
1774         while (!list_empty(&dev_priv->mm.request_list)) {
1775                 struct drm_i915_gem_request *request;
1776                 uint32_t retiring_seqno;
1777
1778                 request = list_first_entry(&dev_priv->mm.request_list,
1779                                            struct drm_i915_gem_request,
1780                                            list);
1781                 retiring_seqno = request->seqno;
1782
1783                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1784                     atomic_read(&dev_priv->mm.wedged)) {
1785                         i915_gem_retire_request(dev, request);
1786
1787                         list_del(&request->list);
1788                         list_del(&request->client_list);
1789                         kfree(request);
1790                 } else
1791                         break;
1792         }
1793
1794         if (unlikely (dev_priv->trace_irq_seqno &&
1795                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1796                 i915_user_irq_put(dev);
1797                 dev_priv->trace_irq_seqno = 0;
1798         }
1799 }
1800
1801 void
1802 i915_gem_retire_work_handler(struct work_struct *work)
1803 {
1804         drm_i915_private_t *dev_priv;
1805         struct drm_device *dev;
1806
1807         dev_priv = container_of(work, drm_i915_private_t,
1808                                 mm.retire_work.work);
1809         dev = dev_priv->dev;
1810
1811         mutex_lock(&dev->struct_mutex);
1812         i915_gem_retire_requests(dev);
1813         if (!dev_priv->mm.suspended &&
1814             !list_empty(&dev_priv->mm.request_list))
1815                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1816         mutex_unlock(&dev->struct_mutex);
1817 }
1818
1819 int
1820 i915_do_wait_request(struct drm_device *dev, uint32_t seqno, int interruptible)
1821 {
1822         drm_i915_private_t *dev_priv = dev->dev_private;
1823         u32 ier;
1824         int ret = 0;
1825
1826         BUG_ON(seqno == 0);
1827
1828         if (atomic_read(&dev_priv->mm.wedged))
1829                 return -EIO;
1830
1831         if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
1832                 if (HAS_PCH_SPLIT(dev))
1833                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1834                 else
1835                         ier = I915_READ(IER);
1836                 if (!ier) {
1837                         DRM_ERROR("something (likely vbetool) disabled "
1838                                   "interrupts, re-enabling\n");
1839                         i915_driver_irq_preinstall(dev);
1840                         i915_driver_irq_postinstall(dev);
1841                 }
1842
1843                 trace_i915_gem_request_wait_begin(dev, seqno);
1844
1845                 dev_priv->mm.waiting_gem_seqno = seqno;
1846                 i915_user_irq_get(dev);
1847                 if (interruptible)
1848                         ret = wait_event_interruptible(dev_priv->irq_queue,
1849                                 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1850                                 atomic_read(&dev_priv->mm.wedged));
1851                 else
1852                         wait_event(dev_priv->irq_queue,
1853                                 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1854                                 atomic_read(&dev_priv->mm.wedged));
1855
1856                 i915_user_irq_put(dev);
1857                 dev_priv->mm.waiting_gem_seqno = 0;
1858
1859                 trace_i915_gem_request_wait_end(dev, seqno);
1860         }
1861         if (atomic_read(&dev_priv->mm.wedged))
1862                 ret = -EIO;
1863
1864         if (ret && ret != -ERESTARTSYS)
1865                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1866                           __func__, ret, seqno, i915_get_gem_seqno(dev));
1867
1868         /* Directly dispatch request retiring.  While we have the work queue
1869          * to handle this, the waiter on a request often wants an associated
1870          * buffer to have made it to the inactive list, and we would need
1871          * a separate wait queue to handle that.
1872          */
1873         if (ret == 0)
1874                 i915_gem_retire_requests(dev);
1875
1876         return ret;
1877 }
1878
1879 /**
1880  * Waits for a sequence number to be signaled, and cleans up the
1881  * request and object lists appropriately for that event.
1882  */
1883 static int
1884 i915_wait_request(struct drm_device *dev, uint32_t seqno)
1885 {
1886         return i915_do_wait_request(dev, seqno, 1);
1887 }
1888
1889 static void
1890 i915_gem_flush(struct drm_device *dev,
1891                uint32_t invalidate_domains,
1892                uint32_t flush_domains)
1893 {
1894         drm_i915_private_t *dev_priv = dev->dev_private;
1895         uint32_t cmd;
1896         RING_LOCALS;
1897
1898 #if WATCH_EXEC
1899         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1900                   invalidate_domains, flush_domains);
1901 #endif
1902         trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
1903                                      invalidate_domains, flush_domains);
1904
1905         if (flush_domains & I915_GEM_DOMAIN_CPU)
1906                 drm_agp_chipset_flush(dev);
1907
1908         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
1909                 /*
1910                  * read/write caches:
1911                  *
1912                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1913                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
1914                  * also flushed at 2d versus 3d pipeline switches.
1915                  *
1916                  * read-only caches:
1917                  *
1918                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1919                  * MI_READ_FLUSH is set, and is always flushed on 965.
1920                  *
1921                  * I915_GEM_DOMAIN_COMMAND may not exist?
1922                  *
1923                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1924                  * invalidated when MI_EXE_FLUSH is set.
1925                  *
1926                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1927                  * invalidated with every MI_FLUSH.
1928                  *
1929                  * TLBs:
1930                  *
1931                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1932                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1933                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1934                  * are flushed at any MI_FLUSH.
1935                  */
1936
1937                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1938                 if ((invalidate_domains|flush_domains) &
1939                     I915_GEM_DOMAIN_RENDER)
1940                         cmd &= ~MI_NO_WRITE_FLUSH;
1941                 if (!IS_I965G(dev)) {
1942                         /*
1943                          * On the 965, the sampler cache always gets flushed
1944                          * and this bit is reserved.
1945                          */
1946                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1947                                 cmd |= MI_READ_FLUSH;
1948                 }
1949                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1950                         cmd |= MI_EXE_FLUSH;
1951
1952 #if WATCH_EXEC
1953                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1954 #endif
1955                 BEGIN_LP_RING(2);
1956                 OUT_RING(cmd);
1957                 OUT_RING(MI_NOOP);
1958                 ADVANCE_LP_RING();
1959         }
1960 }
1961
1962 /**
1963  * Ensures that all rendering to the object has completed and the object is
1964  * safe to unbind from the GTT or access from the CPU.
1965  */
1966 static int
1967 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1968 {
1969         struct drm_device *dev = obj->dev;
1970         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1971         int ret;
1972
1973         /* This function only exists to support waiting for existing rendering,
1974          * not for emitting required flushes.
1975          */
1976         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1977
1978         /* If there is rendering queued on the buffer being evicted, wait for
1979          * it.
1980          */
1981         if (obj_priv->active) {
1982 #if WATCH_BUF
1983                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1984                           __func__, obj, obj_priv->last_rendering_seqno);
1985 #endif
1986                 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1987                 if (ret != 0)
1988                         return ret;
1989         }
1990
1991         return 0;
1992 }
1993
1994 /**
1995  * Unbinds an object from the GTT aperture.
1996  */
1997 int
1998 i915_gem_object_unbind(struct drm_gem_object *obj)
1999 {
2000         struct drm_device *dev = obj->dev;
2001         drm_i915_private_t *dev_priv = dev->dev_private;
2002         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2003         int ret = 0;
2004
2005 #if WATCH_BUF
2006         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2007         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2008 #endif
2009         if (obj_priv->gtt_space == NULL)
2010                 return 0;
2011
2012         if (obj_priv->pin_count != 0) {
2013                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2014                 return -EINVAL;
2015         }
2016
2017         /* blow away mappings if mapped through GTT */
2018         i915_gem_release_mmap(obj);
2019
2020         /* Move the object to the CPU domain to ensure that
2021          * any possible CPU writes while it's not in the GTT
2022          * are flushed when we go to remap it. This will
2023          * also ensure that all pending GPU writes are finished
2024          * before we unbind.
2025          */
2026         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2027         if (ret) {
2028                 if (ret != -ERESTARTSYS)
2029                         DRM_ERROR("set_domain failed: %d\n", ret);
2030                 return ret;
2031         }
2032
2033         BUG_ON(obj_priv->active);
2034
2035         /* release the fence reg _after_ flushing */
2036         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2037                 i915_gem_clear_fence_reg(obj);
2038
2039         if (obj_priv->agp_mem != NULL) {
2040                 drm_unbind_agp(obj_priv->agp_mem);
2041                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2042                 obj_priv->agp_mem = NULL;
2043         }
2044
2045         i915_gem_object_put_pages(obj);
2046         BUG_ON(obj_priv->pages_refcount);
2047
2048         if (obj_priv->gtt_space) {
2049                 atomic_dec(&dev->gtt_count);
2050                 atomic_sub(obj->size, &dev->gtt_memory);
2051
2052                 drm_mm_put_block(obj_priv->gtt_space);
2053                 obj_priv->gtt_space = NULL;
2054         }
2055
2056         /* Remove ourselves from the LRU list if present. */
2057         spin_lock(&dev_priv->mm.active_list_lock);
2058         if (!list_empty(&obj_priv->list))
2059                 list_del_init(&obj_priv->list);
2060         spin_unlock(&dev_priv->mm.active_list_lock);
2061
2062         if (i915_gem_object_is_purgeable(obj_priv))
2063                 i915_gem_object_truncate(obj);
2064
2065         trace_i915_gem_object_unbind(obj);
2066
2067         return 0;
2068 }
2069
2070 static struct drm_gem_object *
2071 i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2072 {
2073         drm_i915_private_t *dev_priv = dev->dev_private;
2074         struct drm_i915_gem_object *obj_priv;
2075         struct drm_gem_object *best = NULL;
2076         struct drm_gem_object *first = NULL;
2077
2078         /* Try to find the smallest clean object */
2079         list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
2080                 struct drm_gem_object *obj = obj_priv->obj;
2081                 if (obj->size >= min_size) {
2082                         if ((!obj_priv->dirty ||
2083                              i915_gem_object_is_purgeable(obj_priv)) &&
2084                             (!best || obj->size < best->size)) {
2085                                 best = obj;
2086                                 if (best->size == min_size)
2087                                         return best;
2088                         }
2089                         if (!first)
2090                             first = obj;
2091                 }
2092         }
2093
2094         return best ? best : first;
2095 }
2096
2097 static int
2098 i915_gpu_idle(struct drm_device *dev)
2099 {
2100         drm_i915_private_t *dev_priv = dev->dev_private;
2101         bool lists_empty;
2102         uint32_t seqno;
2103
2104         spin_lock(&dev_priv->mm.active_list_lock);
2105         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
2106                       list_empty(&dev_priv->mm.active_list);
2107         spin_unlock(&dev_priv->mm.active_list_lock);
2108
2109         if (lists_empty)
2110                 return 0;
2111
2112         /* Flush everything onto the inactive list. */
2113         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2114         seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
2115         if (seqno == 0)
2116                 return -ENOMEM;
2117
2118         return i915_wait_request(dev, seqno);
2119 }
2120
2121 static int
2122 i915_gem_evict_everything(struct drm_device *dev)
2123 {
2124         drm_i915_private_t *dev_priv = dev->dev_private;
2125         int ret;
2126         bool lists_empty;
2127
2128         spin_lock(&dev_priv->mm.active_list_lock);
2129         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2130                        list_empty(&dev_priv->mm.flushing_list) &&
2131                        list_empty(&dev_priv->mm.active_list));
2132         spin_unlock(&dev_priv->mm.active_list_lock);
2133
2134         if (lists_empty)
2135                 return -ENOSPC;
2136
2137         /* Flush everything (on to the inactive lists) and evict */
2138         ret = i915_gpu_idle(dev);
2139         if (ret)
2140                 return ret;
2141
2142         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2143
2144         ret = i915_gem_evict_from_inactive_list(dev);
2145         if (ret)
2146                 return ret;
2147
2148         spin_lock(&dev_priv->mm.active_list_lock);
2149         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2150                        list_empty(&dev_priv->mm.flushing_list) &&
2151                        list_empty(&dev_priv->mm.active_list));
2152         spin_unlock(&dev_priv->mm.active_list_lock);
2153         BUG_ON(!lists_empty);
2154
2155         return 0;
2156 }
2157
2158 static int
2159 i915_gem_evict_something(struct drm_device *dev, int min_size)
2160 {
2161         drm_i915_private_t *dev_priv = dev->dev_private;
2162         struct drm_gem_object *obj;
2163         int ret;
2164
2165         for (;;) {
2166                 i915_gem_retire_requests(dev);
2167
2168                 /* If there's an inactive buffer available now, grab it
2169                  * and be done.
2170                  */
2171                 obj = i915_gem_find_inactive_object(dev, min_size);
2172                 if (obj) {
2173                         struct drm_i915_gem_object *obj_priv;
2174
2175 #if WATCH_LRU
2176                         DRM_INFO("%s: evicting %p\n", __func__, obj);
2177 #endif
2178                         obj_priv = obj->driver_private;
2179                         BUG_ON(obj_priv->pin_count != 0);
2180                         BUG_ON(obj_priv->active);
2181
2182                         /* Wait on the rendering and unbind the buffer. */
2183                         return i915_gem_object_unbind(obj);
2184                 }
2185
2186                 /* If we didn't get anything, but the ring is still processing
2187                  * things, wait for the next to finish and hopefully leave us
2188                  * a buffer to evict.
2189                  */
2190                 if (!list_empty(&dev_priv->mm.request_list)) {
2191                         struct drm_i915_gem_request *request;
2192
2193                         request = list_first_entry(&dev_priv->mm.request_list,
2194                                                    struct drm_i915_gem_request,
2195                                                    list);
2196
2197                         ret = i915_wait_request(dev, request->seqno);
2198                         if (ret)
2199                                 return ret;
2200
2201                         continue;
2202                 }
2203
2204                 /* If we didn't have anything on the request list but there
2205                  * are buffers awaiting a flush, emit one and try again.
2206                  * When we wait on it, those buffers waiting for that flush
2207                  * will get moved to inactive.
2208                  */
2209                 if (!list_empty(&dev_priv->mm.flushing_list)) {
2210                         struct drm_i915_gem_object *obj_priv;
2211
2212                         /* Find an object that we can immediately reuse */
2213                         list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
2214                                 obj = obj_priv->obj;
2215                                 if (obj->size >= min_size)
2216                                         break;
2217
2218                                 obj = NULL;
2219                         }
2220
2221                         if (obj != NULL) {
2222                                 uint32_t seqno;
2223
2224                                 i915_gem_flush(dev,
2225                                                obj->write_domain,
2226                                                obj->write_domain);
2227                                 seqno = i915_add_request(dev, NULL, obj->write_domain);
2228                                 if (seqno == 0)
2229                                         return -ENOMEM;
2230
2231                                 ret = i915_wait_request(dev, seqno);
2232                                 if (ret)
2233                                         return ret;
2234
2235                                 continue;
2236                         }
2237                 }
2238
2239                 /* If we didn't do any of the above, there's no single buffer
2240                  * large enough to swap out for the new one, so just evict
2241                  * everything and start again. (This should be rare.)
2242                  */
2243                 if (!list_empty (&dev_priv->mm.inactive_list))
2244                         return i915_gem_evict_from_inactive_list(dev);
2245                 else
2246                         return i915_gem_evict_everything(dev);
2247         }
2248 }
2249
2250 int
2251 i915_gem_object_get_pages(struct drm_gem_object *obj,
2252                           gfp_t gfpmask)
2253 {
2254         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2255         int page_count, i;
2256         struct address_space *mapping;
2257         struct inode *inode;
2258         struct page *page;
2259         int ret;
2260
2261         if (obj_priv->pages_refcount++ != 0)
2262                 return 0;
2263
2264         /* Get the list of pages out of our struct file.  They'll be pinned
2265          * at this point until we release them.
2266          */
2267         page_count = obj->size / PAGE_SIZE;
2268         BUG_ON(obj_priv->pages != NULL);
2269         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2270         if (obj_priv->pages == NULL) {
2271                 obj_priv->pages_refcount--;
2272                 return -ENOMEM;
2273         }
2274
2275         inode = obj->filp->f_path.dentry->d_inode;
2276         mapping = inode->i_mapping;
2277         for (i = 0; i < page_count; i++) {
2278                 page = read_cache_page_gfp(mapping, i,
2279                                            mapping_gfp_mask (mapping) |
2280                                            __GFP_COLD |
2281                                            gfpmask);
2282                 if (IS_ERR(page)) {
2283                         ret = PTR_ERR(page);
2284                         i915_gem_object_put_pages(obj);
2285                         return ret;
2286                 }
2287                 obj_priv->pages[i] = page;
2288         }
2289
2290         if (obj_priv->tiling_mode != I915_TILING_NONE)
2291                 i915_gem_object_do_bit_17_swizzle(obj);
2292
2293         return 0;
2294 }
2295
2296 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2297 {
2298         struct drm_gem_object *obj = reg->obj;
2299         struct drm_device *dev = obj->dev;
2300         drm_i915_private_t *dev_priv = dev->dev_private;
2301         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2302         int regnum = obj_priv->fence_reg;
2303         uint64_t val;
2304
2305         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2306                     0xfffff000) << 32;
2307         val |= obj_priv->gtt_offset & 0xfffff000;
2308         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2309                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2310
2311         if (obj_priv->tiling_mode == I915_TILING_Y)
2312                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2313         val |= I965_FENCE_REG_VALID;
2314
2315         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2316 }
2317
2318 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2319 {
2320         struct drm_gem_object *obj = reg->obj;
2321         struct drm_device *dev = obj->dev;
2322         drm_i915_private_t *dev_priv = dev->dev_private;
2323         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2324         int regnum = obj_priv->fence_reg;
2325         uint64_t val;
2326
2327         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2328                     0xfffff000) << 32;
2329         val |= obj_priv->gtt_offset & 0xfffff000;
2330         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2331         if (obj_priv->tiling_mode == I915_TILING_Y)
2332                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2333         val |= I965_FENCE_REG_VALID;
2334
2335         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2336 }
2337
2338 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2339 {
2340         struct drm_gem_object *obj = reg->obj;
2341         struct drm_device *dev = obj->dev;
2342         drm_i915_private_t *dev_priv = dev->dev_private;
2343         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2344         int regnum = obj_priv->fence_reg;
2345         int tile_width;
2346         uint32_t fence_reg, val;
2347         uint32_t pitch_val;
2348
2349         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2350             (obj_priv->gtt_offset & (obj->size - 1))) {
2351                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2352                      __func__, obj_priv->gtt_offset, obj->size);
2353                 return;
2354         }
2355
2356         if (obj_priv->tiling_mode == I915_TILING_Y &&
2357             HAS_128_BYTE_Y_TILING(dev))
2358                 tile_width = 128;
2359         else
2360                 tile_width = 512;
2361
2362         /* Note: pitch better be a power of two tile widths */
2363         pitch_val = obj_priv->stride / tile_width;
2364         pitch_val = ffs(pitch_val) - 1;
2365
2366         val = obj_priv->gtt_offset;
2367         if (obj_priv->tiling_mode == I915_TILING_Y)
2368                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2369         val |= I915_FENCE_SIZE_BITS(obj->size);
2370         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2371         val |= I830_FENCE_REG_VALID;
2372
2373         if (regnum < 8)
2374                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2375         else
2376                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2377         I915_WRITE(fence_reg, val);
2378 }
2379
2380 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2381 {
2382         struct drm_gem_object *obj = reg->obj;
2383         struct drm_device *dev = obj->dev;
2384         drm_i915_private_t *dev_priv = dev->dev_private;
2385         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2386         int regnum = obj_priv->fence_reg;
2387         uint32_t val;
2388         uint32_t pitch_val;
2389         uint32_t fence_size_bits;
2390
2391         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2392             (obj_priv->gtt_offset & (obj->size - 1))) {
2393                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2394                      __func__, obj_priv->gtt_offset);
2395                 return;
2396         }
2397
2398         pitch_val = obj_priv->stride / 128;
2399         pitch_val = ffs(pitch_val) - 1;
2400         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2401
2402         val = obj_priv->gtt_offset;
2403         if (obj_priv->tiling_mode == I915_TILING_Y)
2404                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2405         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2406         WARN_ON(fence_size_bits & ~0x00000f00);
2407         val |= fence_size_bits;
2408         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2409         val |= I830_FENCE_REG_VALID;
2410
2411         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2412 }
2413
2414 static int i915_find_fence_reg(struct drm_device *dev)
2415 {
2416         struct drm_i915_fence_reg *reg = NULL;
2417         struct drm_i915_gem_object *obj_priv = NULL;
2418         struct drm_i915_private *dev_priv = dev->dev_private;
2419         struct drm_gem_object *obj = NULL;
2420         int i, avail, ret;
2421
2422         /* First try to find a free reg */
2423         avail = 0;
2424         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2425                 reg = &dev_priv->fence_regs[i];
2426                 if (!reg->obj)
2427                         return i;
2428
2429                 obj_priv = reg->obj->driver_private;
2430                 if (!obj_priv->pin_count)
2431                     avail++;
2432         }
2433
2434         if (avail == 0)
2435                 return -ENOSPC;
2436
2437         /* None available, try to steal one or wait for a user to finish */
2438         i = I915_FENCE_REG_NONE;
2439         list_for_each_entry(obj_priv, &dev_priv->mm.fence_list,
2440                             fence_list) {
2441                 obj = obj_priv->obj;
2442
2443                 if (obj_priv->pin_count)
2444                         continue;
2445
2446                 /* found one! */
2447                 i = obj_priv->fence_reg;
2448                 break;
2449         }
2450
2451         BUG_ON(i == I915_FENCE_REG_NONE);
2452
2453         /* We only have a reference on obj from the active list. put_fence_reg
2454          * might drop that one, causing a use-after-free in it. So hold a
2455          * private reference to obj like the other callers of put_fence_reg
2456          * (set_tiling ioctl) do. */
2457         drm_gem_object_reference(obj);
2458         ret = i915_gem_object_put_fence_reg(obj);
2459         drm_gem_object_unreference(obj);
2460         if (ret != 0)
2461                 return ret;
2462
2463         return i;
2464 }
2465
2466 /**
2467  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2468  * @obj: object to map through a fence reg
2469  *
2470  * When mapping objects through the GTT, userspace wants to be able to write
2471  * to them without having to worry about swizzling if the object is tiled.
2472  *
2473  * This function walks the fence regs looking for a free one for @obj,
2474  * stealing one if it can't find any.
2475  *
2476  * It then sets up the reg based on the object's properties: address, pitch
2477  * and tiling format.
2478  */
2479 int
2480 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2481 {
2482         struct drm_device *dev = obj->dev;
2483         struct drm_i915_private *dev_priv = dev->dev_private;
2484         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2485         struct drm_i915_fence_reg *reg = NULL;
2486         int ret;
2487
2488         /* Just update our place in the LRU if our fence is getting used. */
2489         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2490                 list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2491                 return 0;
2492         }
2493
2494         switch (obj_priv->tiling_mode) {
2495         case I915_TILING_NONE:
2496                 WARN(1, "allocating a fence for non-tiled object?\n");
2497                 break;
2498         case I915_TILING_X:
2499                 if (!obj_priv->stride)
2500                         return -EINVAL;
2501                 WARN((obj_priv->stride & (512 - 1)),
2502                      "object 0x%08x is X tiled but has non-512B pitch\n",
2503                      obj_priv->gtt_offset);
2504                 break;
2505         case I915_TILING_Y:
2506                 if (!obj_priv->stride)
2507                         return -EINVAL;
2508                 WARN((obj_priv->stride & (128 - 1)),
2509                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2510                      obj_priv->gtt_offset);
2511                 break;
2512         }
2513
2514         ret = i915_find_fence_reg(dev);
2515         if (ret < 0)
2516                 return ret;
2517
2518         obj_priv->fence_reg = ret;
2519         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2520         list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2521
2522         reg->obj = obj;
2523
2524         if (IS_GEN6(dev))
2525                 sandybridge_write_fence_reg(reg);
2526         else if (IS_I965G(dev))
2527                 i965_write_fence_reg(reg);
2528         else if (IS_I9XX(dev))
2529                 i915_write_fence_reg(reg);
2530         else
2531                 i830_write_fence_reg(reg);
2532
2533         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2534                         obj_priv->tiling_mode);
2535
2536         return 0;
2537 }
2538
2539 /**
2540  * i915_gem_clear_fence_reg - clear out fence register info
2541  * @obj: object to clear
2542  *
2543  * Zeroes out the fence register itself and clears out the associated
2544  * data structures in dev_priv and obj_priv.
2545  */
2546 static void
2547 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2548 {
2549         struct drm_device *dev = obj->dev;
2550         drm_i915_private_t *dev_priv = dev->dev_private;
2551         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2552
2553         if (IS_GEN6(dev)) {
2554                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2555                              (obj_priv->fence_reg * 8), 0);
2556         } else if (IS_I965G(dev)) {
2557                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2558         } else {
2559                 uint32_t fence_reg;
2560
2561                 if (obj_priv->fence_reg < 8)
2562                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2563                 else
2564                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2565                                                        8) * 4;
2566
2567                 I915_WRITE(fence_reg, 0);
2568         }
2569
2570         dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2571         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2572         list_del_init(&obj_priv->fence_list);
2573 }
2574
2575 /**
2576  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2577  * to the buffer to finish, and then resets the fence register.
2578  * @obj: tiled object holding a fence register.
2579  *
2580  * Zeroes out the fence register itself and clears out the associated
2581  * data structures in dev_priv and obj_priv.
2582  */
2583 int
2584 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2585 {
2586         struct drm_device *dev = obj->dev;
2587         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2588
2589         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2590                 return 0;
2591
2592         /* If we've changed tiling, GTT-mappings of the object
2593          * need to re-fault to ensure that the correct fence register
2594          * setup is in place.
2595          */
2596         i915_gem_release_mmap(obj);
2597
2598         /* On the i915, GPU access to tiled buffers is via a fence,
2599          * therefore we must wait for any outstanding access to complete
2600          * before clearing the fence.
2601          */
2602         if (!IS_I965G(dev)) {
2603                 int ret;
2604
2605                 i915_gem_object_flush_gpu_write_domain(obj);
2606                 ret = i915_gem_object_wait_rendering(obj);
2607                 if (ret != 0)
2608                         return ret;
2609         }
2610
2611         i915_gem_object_flush_gtt_write_domain(obj);
2612         i915_gem_clear_fence_reg (obj);
2613
2614         return 0;
2615 }
2616
2617 /**
2618  * Finds free space in the GTT aperture and binds the object there.
2619  */
2620 static int
2621 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2622 {
2623         struct drm_device *dev = obj->dev;
2624         drm_i915_private_t *dev_priv = dev->dev_private;
2625         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2626         struct drm_mm_node *free_space;
2627         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2628         int ret;
2629
2630         if (obj_priv->madv != I915_MADV_WILLNEED) {
2631                 DRM_ERROR("Attempting to bind a purgeable object\n");
2632                 return -EINVAL;
2633         }
2634
2635         if (alignment == 0)
2636                 alignment = i915_gem_get_gtt_alignment(obj);
2637         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2638                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2639                 return -EINVAL;
2640         }
2641
2642  search_free:
2643         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2644                                         obj->size, alignment, 0);
2645         if (free_space != NULL) {
2646                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2647                                                        alignment);
2648                 if (obj_priv->gtt_space != NULL) {
2649                         obj_priv->gtt_space->private = obj;
2650                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2651                 }
2652         }
2653         if (obj_priv->gtt_space == NULL) {
2654                 /* If the gtt is empty and we're still having trouble
2655                  * fitting our object in, we're out of memory.
2656                  */
2657 #if WATCH_LRU
2658                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2659 #endif
2660                 ret = i915_gem_evict_something(dev, obj->size);
2661                 if (ret)
2662                         return ret;
2663
2664                 goto search_free;
2665         }
2666
2667 #if WATCH_BUF
2668         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2669                  obj->size, obj_priv->gtt_offset);
2670 #endif
2671         ret = i915_gem_object_get_pages(obj, gfpmask);
2672         if (ret) {
2673                 drm_mm_put_block(obj_priv->gtt_space);
2674                 obj_priv->gtt_space = NULL;
2675
2676                 if (ret == -ENOMEM) {
2677                         /* first try to clear up some space from the GTT */
2678                         ret = i915_gem_evict_something(dev, obj->size);
2679                         if (ret) {
2680                                 /* now try to shrink everyone else */
2681                                 if (gfpmask) {
2682                                         gfpmask = 0;
2683                                         goto search_free;
2684                                 }
2685
2686                                 return ret;
2687                         }
2688
2689                         goto search_free;
2690                 }
2691
2692                 return ret;
2693         }
2694
2695         /* Create an AGP memory structure pointing at our pages, and bind it
2696          * into the GTT.
2697          */
2698         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2699                                                obj_priv->pages,
2700                                                obj->size >> PAGE_SHIFT,
2701                                                obj_priv->gtt_offset,
2702                                                obj_priv->agp_type);
2703         if (obj_priv->agp_mem == NULL) {
2704                 i915_gem_object_put_pages(obj);
2705                 drm_mm_put_block(obj_priv->gtt_space);
2706                 obj_priv->gtt_space = NULL;
2707
2708                 ret = i915_gem_evict_something(dev, obj->size);
2709                 if (ret)
2710                         return ret;
2711
2712                 goto search_free;
2713         }
2714         atomic_inc(&dev->gtt_count);
2715         atomic_add(obj->size, &dev->gtt_memory);
2716
2717         /* Assert that the object is not currently in any GPU domain. As it
2718          * wasn't in the GTT, there shouldn't be any way it could have been in
2719          * a GPU cache
2720          */
2721         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2722         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2723
2724         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2725
2726         return 0;
2727 }
2728
2729 void
2730 i915_gem_clflush_object(struct drm_gem_object *obj)
2731 {
2732         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
2733
2734         /* If we don't have a page list set up, then we're not pinned
2735          * to GPU, and we can ignore the cache flush because it'll happen
2736          * again at bind time.
2737          */
2738         if (obj_priv->pages == NULL)
2739                 return;
2740
2741         trace_i915_gem_object_clflush(obj);
2742
2743         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2744 }
2745
2746 /** Flushes any GPU write domain for the object if it's dirty. */
2747 static void
2748 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2749 {
2750         struct drm_device *dev = obj->dev;
2751         uint32_t old_write_domain;
2752
2753         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2754                 return;
2755
2756         /* Queue the GPU write cache flushing we need. */
2757         old_write_domain = obj->write_domain;
2758         i915_gem_flush(dev, 0, obj->write_domain);
2759         (void) i915_add_request(dev, NULL, obj->write_domain);
2760         BUG_ON(obj->write_domain);
2761
2762         trace_i915_gem_object_change_domain(obj,
2763                                             obj->read_domains,
2764                                             old_write_domain);
2765 }
2766
2767 /** Flushes the GTT write domain for the object if it's dirty. */
2768 static void
2769 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2770 {
2771         uint32_t old_write_domain;
2772
2773         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2774                 return;
2775
2776         /* No actual flushing is required for the GTT write domain.   Writes
2777          * to it immediately go to main memory as far as we know, so there's
2778          * no chipset flush.  It also doesn't land in render cache.
2779          */
2780         old_write_domain = obj->write_domain;
2781         obj->write_domain = 0;
2782
2783         trace_i915_gem_object_change_domain(obj,
2784                                             obj->read_domains,
2785                                             old_write_domain);
2786 }
2787
2788 /** Flushes the CPU write domain for the object if it's dirty. */
2789 static void
2790 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2791 {
2792         struct drm_device *dev = obj->dev;
2793         uint32_t old_write_domain;
2794
2795         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2796                 return;
2797
2798         i915_gem_clflush_object(obj);
2799         drm_agp_chipset_flush(dev);
2800         old_write_domain = obj->write_domain;
2801         obj->write_domain = 0;
2802
2803         trace_i915_gem_object_change_domain(obj,
2804                                             obj->read_domains,
2805                                             old_write_domain);
2806 }
2807
2808 void
2809 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2810 {
2811         switch (obj->write_domain) {
2812         case I915_GEM_DOMAIN_GTT:
2813                 i915_gem_object_flush_gtt_write_domain(obj);
2814                 break;
2815         case I915_GEM_DOMAIN_CPU:
2816                 i915_gem_object_flush_cpu_write_domain(obj);
2817                 break;
2818         default:
2819                 i915_gem_object_flush_gpu_write_domain(obj);
2820                 break;
2821         }
2822 }
2823
2824 /**
2825  * Moves a single object to the GTT read, and possibly write domain.
2826  *
2827  * This function returns when the move is complete, including waiting on
2828  * flushes to occur.
2829  */
2830 int
2831 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2832 {
2833         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2834         uint32_t old_write_domain, old_read_domains;
2835         int ret;
2836
2837         /* Not valid to be called on unbound objects. */
2838         if (obj_priv->gtt_space == NULL)
2839                 return -EINVAL;
2840
2841         i915_gem_object_flush_gpu_write_domain(obj);
2842         /* Wait on any GPU rendering and flushing to occur. */
2843         ret = i915_gem_object_wait_rendering(obj);
2844         if (ret != 0)
2845                 return ret;
2846
2847         old_write_domain = obj->write_domain;
2848         old_read_domains = obj->read_domains;
2849
2850         /* If we're writing through the GTT domain, then CPU and GPU caches
2851          * will need to be invalidated at next use.
2852          */
2853         if (write)
2854                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2855
2856         i915_gem_object_flush_cpu_write_domain(obj);
2857
2858         /* It should now be out of any other write domains, and we can update
2859          * the domain values for our changes.
2860          */
2861         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2862         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2863         if (write) {
2864                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2865                 obj_priv->dirty = 1;
2866         }
2867
2868         trace_i915_gem_object_change_domain(obj,
2869                                             old_read_domains,
2870                                             old_write_domain);
2871
2872         return 0;
2873 }
2874
2875 /*
2876  * Prepare buffer for display plane. Use uninterruptible for possible flush
2877  * wait, as in modesetting process we're not supposed to be interrupted.
2878  */
2879 int
2880 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2881 {
2882         struct drm_device *dev = obj->dev;
2883         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2884         uint32_t old_write_domain, old_read_domains;
2885         int ret;
2886
2887         /* Not valid to be called on unbound objects. */
2888         if (obj_priv->gtt_space == NULL)
2889                 return -EINVAL;
2890
2891         i915_gem_object_flush_gpu_write_domain(obj);
2892
2893         /* Wait on any GPU rendering and flushing to occur. */
2894         if (obj_priv->active) {
2895 #if WATCH_BUF
2896                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2897                           __func__, obj, obj_priv->last_rendering_seqno);
2898 #endif
2899                 ret = i915_do_wait_request(dev, obj_priv->last_rendering_seqno, 0);
2900                 if (ret != 0)
2901                         return ret;
2902         }
2903
2904         old_write_domain = obj->write_domain;
2905         old_read_domains = obj->read_domains;
2906
2907         obj->read_domains &= I915_GEM_DOMAIN_GTT;
2908
2909         i915_gem_object_flush_cpu_write_domain(obj);
2910
2911         /* It should now be out of any other write domains, and we can update
2912          * the domain values for our changes.
2913          */
2914         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2915         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2916         obj->write_domain = I915_GEM_DOMAIN_GTT;
2917         obj_priv->dirty = 1;
2918
2919         trace_i915_gem_object_change_domain(obj,
2920                                             old_read_domains,
2921                                             old_write_domain);
2922
2923         return 0;
2924 }
2925
2926 /**
2927  * Moves a single object to the CPU read, and possibly write domain.
2928  *
2929  * This function returns when the move is complete, including waiting on
2930  * flushes to occur.
2931  */
2932 static int
2933 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2934 {
2935         uint32_t old_write_domain, old_read_domains;
2936         int ret;
2937
2938         i915_gem_object_flush_gpu_write_domain(obj);
2939         /* Wait on any GPU rendering and flushing to occur. */
2940         ret = i915_gem_object_wait_rendering(obj);
2941         if (ret != 0)
2942                 return ret;
2943
2944         i915_gem_object_flush_gtt_write_domain(obj);
2945
2946         /* If we have a partially-valid cache of the object in the CPU,
2947          * finish invalidating it and free the per-page flags.
2948          */
2949         i915_gem_object_set_to_full_cpu_read_domain(obj);
2950
2951         old_write_domain = obj->write_domain;
2952         old_read_domains = obj->read_domains;
2953
2954         /* Flush the CPU cache if it's still invalid. */
2955         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2956                 i915_gem_clflush_object(obj);
2957
2958                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2959         }
2960
2961         /* It should now be out of any other write domains, and we can update
2962          * the domain values for our changes.
2963          */
2964         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2965
2966         /* If we're writing through the CPU, then the GPU read domains will
2967          * need to be invalidated at next use.
2968          */
2969         if (write) {
2970                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2971                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2972         }
2973
2974         trace_i915_gem_object_change_domain(obj,
2975                                             old_read_domains,
2976                                             old_write_domain);
2977
2978         return 0;
2979 }
2980
2981 /*
2982  * Set the next domain for the specified object. This
2983  * may not actually perform the necessary flushing/invaliding though,
2984  * as that may want to be batched with other set_domain operations
2985  *
2986  * This is (we hope) the only really tricky part of gem. The goal
2987  * is fairly simple -- track which caches hold bits of the object
2988  * and make sure they remain coherent. A few concrete examples may
2989  * help to explain how it works. For shorthand, we use the notation
2990  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2991  * a pair of read and write domain masks.
2992  *
2993  * Case 1: the batch buffer
2994  *
2995  *      1. Allocated
2996  *      2. Written by CPU
2997  *      3. Mapped to GTT
2998  *      4. Read by GPU
2999  *      5. Unmapped from GTT
3000  *      6. Freed
3001  *
3002  *      Let's take these a step at a time
3003  *
3004  *      1. Allocated
3005  *              Pages allocated from the kernel may still have
3006  *              cache contents, so we set them to (CPU, CPU) always.
3007  *      2. Written by CPU (using pwrite)
3008  *              The pwrite function calls set_domain (CPU, CPU) and
3009  *              this function does nothing (as nothing changes)
3010  *      3. Mapped by GTT
3011  *              This function asserts that the object is not
3012  *              currently in any GPU-based read or write domains
3013  *      4. Read by GPU
3014  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3015  *              As write_domain is zero, this function adds in the
3016  *              current read domains (CPU+COMMAND, 0).
3017  *              flush_domains is set to CPU.
3018  *              invalidate_domains is set to COMMAND
3019  *              clflush is run to get data out of the CPU caches
3020  *              then i915_dev_set_domain calls i915_gem_flush to
3021  *              emit an MI_FLUSH and drm_agp_chipset_flush
3022  *      5. Unmapped from GTT
3023  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3024  *              flush_domains and invalidate_domains end up both zero
3025  *              so no flushing/invalidating happens
3026  *      6. Freed
3027  *              yay, done
3028  *
3029  * Case 2: The shared render buffer
3030  *
3031  *      1. Allocated
3032  *      2. Mapped to GTT
3033  *      3. Read/written by GPU
3034  *      4. set_domain to (CPU,CPU)
3035  *      5. Read/written by CPU
3036  *      6. Read/written by GPU
3037  *
3038  *      1. Allocated
3039  *              Same as last example, (CPU, CPU)
3040  *      2. Mapped to GTT
3041  *              Nothing changes (assertions find that it is not in the GPU)
3042  *      3. Read/written by GPU
3043  *              execbuffer calls set_domain (RENDER, RENDER)
3044  *              flush_domains gets CPU
3045  *              invalidate_domains gets GPU
3046  *              clflush (obj)
3047  *              MI_FLUSH and drm_agp_chipset_flush
3048  *      4. set_domain (CPU, CPU)
3049  *              flush_domains gets GPU
3050  *              invalidate_domains gets CPU
3051  *              wait_rendering (obj) to make sure all drawing is complete.
3052  *              This will include an MI_FLUSH to get the data from GPU
3053  *              to memory
3054  *              clflush (obj) to invalidate the CPU cache
3055  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3056  *      5. Read/written by CPU
3057  *              cache lines are loaded and dirtied
3058  *      6. Read written by GPU
3059  *              Same as last GPU access
3060  *
3061  * Case 3: The constant buffer
3062  *
3063  *      1. Allocated
3064  *      2. Written by CPU
3065  *      3. Read by GPU
3066  *      4. Updated (written) by CPU again
3067  *      5. Read by GPU
3068  *
3069  *      1. Allocated
3070  *              (CPU, CPU)
3071  *      2. Written by CPU
3072  *              (CPU, CPU)
3073  *      3. Read by GPU
3074  *              (CPU+RENDER, 0)
3075  *              flush_domains = CPU
3076  *              invalidate_domains = RENDER
3077  *              clflush (obj)
3078  *              MI_FLUSH
3079  *              drm_agp_chipset_flush
3080  *      4. Updated (written) by CPU again
3081  *              (CPU, CPU)
3082  *              flush_domains = 0 (no previous write domain)
3083  *              invalidate_domains = 0 (no new read domains)
3084  *      5. Read by GPU
3085  *              (CPU+RENDER, 0)
3086  *              flush_domains = CPU
3087  *              invalidate_domains = RENDER
3088  *              clflush (obj)
3089  *              MI_FLUSH
3090  *              drm_agp_chipset_flush
3091  */
3092 static void
3093 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3094 {
3095         struct drm_device               *dev = obj->dev;
3096         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
3097         uint32_t                        invalidate_domains = 0;
3098         uint32_t                        flush_domains = 0;
3099         uint32_t                        old_read_domains;
3100
3101         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3102         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
3103
3104         intel_mark_busy(dev, obj);
3105
3106 #if WATCH_BUF
3107         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3108                  __func__, obj,
3109                  obj->read_domains, obj->pending_read_domains,
3110                  obj->write_domain, obj->pending_write_domain);
3111 #endif
3112         /*
3113          * If the object isn't moving to a new write domain,
3114          * let the object stay in multiple read domains
3115          */
3116         if (obj->pending_write_domain == 0)
3117                 obj->pending_read_domains |= obj->read_domains;
3118         else
3119                 obj_priv->dirty = 1;
3120
3121         /*
3122          * Flush the current write domain if
3123          * the new read domains don't match. Invalidate
3124          * any read domains which differ from the old
3125          * write domain
3126          */
3127         if (obj->write_domain &&
3128             obj->write_domain != obj->pending_read_domains) {
3129                 flush_domains |= obj->write_domain;
3130                 invalidate_domains |=
3131                         obj->pending_read_domains & ~obj->write_domain;
3132         }
3133         /*
3134          * Invalidate any read caches which may have
3135          * stale data. That is, any new read domains.
3136          */
3137         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3138         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3139 #if WATCH_BUF
3140                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3141                          __func__, flush_domains, invalidate_domains);
3142 #endif
3143                 i915_gem_clflush_object(obj);
3144         }
3145
3146         old_read_domains = obj->read_domains;
3147
3148         /* The actual obj->write_domain will be updated with
3149          * pending_write_domain after we emit the accumulated flush for all
3150          * of our domain changes in execbuffers (which clears objects'
3151          * write_domains).  So if we have a current write domain that we
3152          * aren't changing, set pending_write_domain to that.
3153          */
3154         if (flush_domains == 0 && obj->pending_write_domain == 0)
3155                 obj->pending_write_domain = obj->write_domain;
3156         obj->read_domains = obj->pending_read_domains;
3157
3158         dev->invalidate_domains |= invalidate_domains;
3159         dev->flush_domains |= flush_domains;
3160 #if WATCH_BUF
3161         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3162                  __func__,
3163                  obj->read_domains, obj->write_domain,
3164                  dev->invalidate_domains, dev->flush_domains);
3165 #endif
3166
3167         trace_i915_gem_object_change_domain(obj,
3168                                             old_read_domains,
3169                                             obj->write_domain);
3170 }
3171
3172 /**
3173  * Moves the object from a partially CPU read to a full one.
3174  *
3175  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3176  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3177  */
3178 static void
3179 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3180 {
3181         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3182
3183         if (!obj_priv->page_cpu_valid)
3184                 return;
3185
3186         /* If we're partially in the CPU read domain, finish moving it in.
3187          */
3188         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3189                 int i;
3190
3191                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3192                         if (obj_priv->page_cpu_valid[i])
3193                                 continue;
3194                         drm_clflush_pages(obj_priv->pages + i, 1);
3195                 }
3196         }
3197
3198         /* Free the page_cpu_valid mappings which are now stale, whether
3199          * or not we've got I915_GEM_DOMAIN_CPU.
3200          */
3201         kfree(obj_priv->page_cpu_valid);
3202         obj_priv->page_cpu_valid = NULL;
3203 }
3204
3205 /**
3206  * Set the CPU read domain on a range of the object.
3207  *
3208  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3209  * not entirely valid.  The page_cpu_valid member of the object flags which
3210  * pages have been flushed, and will be respected by
3211  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3212  * of the whole object.
3213  *
3214  * This function returns when the move is complete, including waiting on
3215  * flushes to occur.
3216  */
3217 static int
3218 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3219                                           uint64_t offset, uint64_t size)
3220 {
3221         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3222         uint32_t old_read_domains;
3223         int i, ret;
3224
3225         if (offset == 0 && size == obj->size)
3226                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3227
3228         i915_gem_object_flush_gpu_write_domain(obj);
3229         /* Wait on any GPU rendering and flushing to occur. */
3230         ret = i915_gem_object_wait_rendering(obj);
3231         if (ret != 0)
3232                 return ret;
3233         i915_gem_object_flush_gtt_write_domain(obj);
3234
3235         /* If we're already fully in the CPU read domain, we're done. */
3236         if (obj_priv->page_cpu_valid == NULL &&
3237             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3238                 return 0;
3239
3240         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3241          * newly adding I915_GEM_DOMAIN_CPU
3242          */
3243         if (obj_priv->page_cpu_valid == NULL) {
3244                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3245                                                    GFP_KERNEL);
3246                 if (obj_priv->page_cpu_valid == NULL)
3247                         return -ENOMEM;
3248         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3249                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3250
3251         /* Flush the cache on any pages that are still invalid from the CPU's
3252          * perspective.
3253          */
3254         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3255              i++) {
3256                 if (obj_priv->page_cpu_valid[i])
3257                         continue;
3258
3259                 drm_clflush_pages(obj_priv->pages + i, 1);
3260
3261                 obj_priv->page_cpu_valid[i] = 1;
3262         }
3263
3264         /* It should now be out of any other write domains, and we can update
3265          * the domain values for our changes.
3266          */
3267         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3268
3269         old_read_domains = obj->read_domains;
3270         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3271
3272         trace_i915_gem_object_change_domain(obj,
3273                                             old_read_domains,
3274                                             obj->write_domain);
3275
3276         return 0;
3277 }
3278
3279 /**
3280  * Pin an object to the GTT and evaluate the relocations landing in it.
3281  */
3282 static int
3283 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3284                                  struct drm_file *file_priv,
3285                                  struct drm_i915_gem_exec_object2 *entry,
3286                                  struct drm_i915_gem_relocation_entry *relocs)
3287 {
3288         struct drm_device *dev = obj->dev;
3289         drm_i915_private_t *dev_priv = dev->dev_private;
3290         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3291         int i, ret;
3292         void __iomem *reloc_page;
3293         bool need_fence;
3294
3295         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3296                      obj_priv->tiling_mode != I915_TILING_NONE;
3297
3298         /* Check fence reg constraints and rebind if necessary */
3299         if (need_fence && !i915_gem_object_fence_offset_ok(obj,
3300             obj_priv->tiling_mode))
3301                 i915_gem_object_unbind(obj);
3302
3303         /* Choose the GTT offset for our buffer and put it there. */
3304         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3305         if (ret)
3306                 return ret;
3307
3308         /*
3309          * Pre-965 chips need a fence register set up in order to
3310          * properly handle blits to/from tiled surfaces.
3311          */
3312         if (need_fence) {
3313                 ret = i915_gem_object_get_fence_reg(obj);
3314                 if (ret != 0) {
3315                         if (ret != -EBUSY && ret != -ERESTARTSYS)
3316                                 DRM_ERROR("Failure to install fence: %d\n",
3317                                           ret);
3318                         i915_gem_object_unpin(obj);
3319                         return ret;
3320                 }
3321         }
3322
3323         entry->offset = obj_priv->gtt_offset;
3324
3325         /* Apply the relocations, using the GTT aperture to avoid cache
3326          * flushing requirements.
3327          */
3328         for (i = 0; i < entry->relocation_count; i++) {
3329                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3330                 struct drm_gem_object *target_obj;
3331                 struct drm_i915_gem_object *target_obj_priv;
3332                 uint32_t reloc_val, reloc_offset;
3333                 uint32_t __iomem *reloc_entry;
3334
3335                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3336                                                    reloc->target_handle);
3337                 if (target_obj == NULL) {
3338                         i915_gem_object_unpin(obj);
3339                         return -EBADF;
3340                 }
3341                 target_obj_priv = target_obj->driver_private;
3342
3343 #if WATCH_RELOC
3344                 DRM_INFO("%s: obj %p offset %08x target %d "
3345                          "read %08x write %08x gtt %08x "
3346                          "presumed %08x delta %08x\n",
3347                          __func__,
3348                          obj,
3349                          (int) reloc->offset,
3350                          (int) reloc->target_handle,
3351                          (int) reloc->read_domains,
3352                          (int) reloc->write_domain,
3353                          (int) target_obj_priv->gtt_offset,
3354                          (int) reloc->presumed_offset,
3355                          reloc->delta);
3356 #endif
3357
3358                 /* The target buffer should have appeared before us in the
3359                  * exec_object list, so it should have a GTT space bound by now.
3360                  */
3361                 if (target_obj_priv->gtt_space == NULL) {
3362                         DRM_ERROR("No GTT space found for object %d\n",
3363                                   reloc->target_handle);
3364                         drm_gem_object_unreference(target_obj);
3365                         i915_gem_object_unpin(obj);
3366                         return -EINVAL;
3367                 }
3368
3369                 /* Validate that the target is in a valid r/w GPU domain */
3370                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3371                         DRM_ERROR("reloc with multiple write domains: "
3372                                   "obj %p target %d offset %d "
3373                                   "read %08x write %08x",
3374                                   obj, reloc->target_handle,
3375                                   (int) reloc->offset,
3376                                   reloc->read_domains,
3377                                   reloc->write_domain);
3378                         return -EINVAL;
3379                 }
3380                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3381                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3382                         DRM_ERROR("reloc with read/write CPU domains: "
3383                                   "obj %p target %d offset %d "
3384                                   "read %08x write %08x",
3385                                   obj, reloc->target_handle,
3386                                   (int) reloc->offset,
3387                                   reloc->read_domains,
3388                                   reloc->write_domain);
3389                         drm_gem_object_unreference(target_obj);
3390                         i915_gem_object_unpin(obj);
3391                         return -EINVAL;
3392                 }
3393                 if (reloc->write_domain && target_obj->pending_write_domain &&
3394                     reloc->write_domain != target_obj->pending_write_domain) {
3395                         DRM_ERROR("Write domain conflict: "
3396                                   "obj %p target %d offset %d "
3397                                   "new %08x old %08x\n",
3398                                   obj, reloc->target_handle,
3399                                   (int) reloc->offset,
3400                                   reloc->write_domain,
3401                                   target_obj->pending_write_domain);
3402                         drm_gem_object_unreference(target_obj);
3403                         i915_gem_object_unpin(obj);
3404                         return -EINVAL;
3405                 }
3406
3407                 target_obj->pending_read_domains |= reloc->read_domains;
3408                 target_obj->pending_write_domain |= reloc->write_domain;
3409
3410                 /* If the relocation already has the right value in it, no
3411                  * more work needs to be done.
3412                  */
3413                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3414                         drm_gem_object_unreference(target_obj);
3415                         continue;
3416                 }
3417
3418                 /* Check that the relocation address is valid... */
3419                 if (reloc->offset > obj->size - 4) {
3420                         DRM_ERROR("Relocation beyond object bounds: "
3421                                   "obj %p target %d offset %d size %d.\n",
3422                                   obj, reloc->target_handle,
3423                                   (int) reloc->offset, (int) obj->size);
3424                         drm_gem_object_unreference(target_obj);
3425                         i915_gem_object_unpin(obj);
3426                         return -EINVAL;
3427                 }
3428                 if (reloc->offset & 3) {
3429                         DRM_ERROR("Relocation not 4-byte aligned: "
3430                                   "obj %p target %d offset %d.\n",
3431                                   obj, reloc->target_handle,
3432                                   (int) reloc->offset);
3433                         drm_gem_object_unreference(target_obj);
3434                         i915_gem_object_unpin(obj);
3435                         return -EINVAL;
3436                 }
3437
3438                 /* and points to somewhere within the target object. */
3439                 if (reloc->delta >= target_obj->size) {
3440                         DRM_ERROR("Relocation beyond target object bounds: "
3441                                   "obj %p target %d delta %d size %d.\n",
3442                                   obj, reloc->target_handle,
3443                                   (int) reloc->delta, (int) target_obj->size);
3444                         drm_gem_object_unreference(target_obj);
3445                         i915_gem_object_unpin(obj);
3446                         return -EINVAL;
3447                 }
3448
3449                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3450                 if (ret != 0) {
3451                         drm_gem_object_unreference(target_obj);
3452                         i915_gem_object_unpin(obj);
3453                         return -EINVAL;
3454                 }
3455
3456                 /* Map the page containing the relocation we're going to
3457                  * perform.
3458                  */
3459                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3460                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3461                                                       (reloc_offset &
3462                                                        ~(PAGE_SIZE - 1)));
3463                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3464                                                    (reloc_offset & (PAGE_SIZE - 1)));
3465                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3466
3467 #if WATCH_BUF
3468                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3469                           obj, (unsigned int) reloc->offset,
3470                           readl(reloc_entry), reloc_val);
3471 #endif
3472                 writel(reloc_val, reloc_entry);
3473                 io_mapping_unmap_atomic(reloc_page);
3474
3475                 /* The updated presumed offset for this entry will be
3476                  * copied back out to the user.
3477                  */
3478                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3479
3480                 drm_gem_object_unreference(target_obj);
3481         }
3482
3483 #if WATCH_BUF
3484         if (0)
3485                 i915_gem_dump_object(obj, 128, __func__, ~0);
3486 #endif
3487         return 0;
3488 }
3489
3490 /** Dispatch a batchbuffer to the ring
3491  */
3492 static int
3493 i915_dispatch_gem_execbuffer(struct drm_device *dev,
3494                               struct drm_i915_gem_execbuffer2 *exec,
3495                               struct drm_clip_rect *cliprects,
3496                               uint64_t exec_offset)
3497 {
3498         drm_i915_private_t *dev_priv = dev->dev_private;
3499         int nbox = exec->num_cliprects;
3500         int i = 0, count;
3501         uint32_t exec_start, exec_len;
3502         RING_LOCALS;
3503
3504         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3505         exec_len = (uint32_t) exec->batch_len;
3506
3507         trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
3508
3509         count = nbox ? nbox : 1;
3510
3511         for (i = 0; i < count; i++) {
3512                 if (i < nbox) {
3513                         int ret = i915_emit_box(dev, cliprects, i,
3514                                                 exec->DR1, exec->DR4);
3515                         if (ret)
3516                                 return ret;
3517                 }
3518
3519                 if (IS_I830(dev) || IS_845G(dev)) {
3520                         BEGIN_LP_RING(4);
3521                         OUT_RING(MI_BATCH_BUFFER);
3522                         OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3523                         OUT_RING(exec_start + exec_len - 4);
3524                         OUT_RING(0);
3525                         ADVANCE_LP_RING();
3526                 } else {
3527                         BEGIN_LP_RING(2);
3528                         if (IS_I965G(dev)) {
3529                                 OUT_RING(MI_BATCH_BUFFER_START |
3530                                          (2 << 6) |
3531                                          MI_BATCH_NON_SECURE_I965);
3532                                 OUT_RING(exec_start);
3533                         } else {
3534                                 OUT_RING(MI_BATCH_BUFFER_START |
3535                                          (2 << 6));
3536                                 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3537                         }
3538                         ADVANCE_LP_RING();
3539                 }
3540         }
3541
3542         /* XXX breadcrumb */
3543         return 0;
3544 }
3545
3546 /* Throttle our rendering by waiting until the ring has completed our requests
3547  * emitted over 20 msec ago.
3548  *
3549  * Note that if we were to use the current jiffies each time around the loop,
3550  * we wouldn't escape the function with any frames outstanding if the time to
3551  * render a frame was over 20ms.
3552  *
3553  * This should get us reasonable parallelism between CPU and GPU but also
3554  * relatively low latency when blocking on a particular request to finish.
3555  */
3556 static int
3557 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3558 {
3559         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3560         int ret = 0;
3561         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3562
3563         mutex_lock(&dev->struct_mutex);
3564         while (!list_empty(&i915_file_priv->mm.request_list)) {
3565                 struct drm_i915_gem_request *request;
3566
3567                 request = list_first_entry(&i915_file_priv->mm.request_list,
3568                                            struct drm_i915_gem_request,
3569                                            client_list);
3570
3571                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3572                         break;
3573
3574                 ret = i915_wait_request(dev, request->seqno);
3575                 if (ret != 0)
3576                         break;
3577         }
3578         mutex_unlock(&dev->struct_mutex);
3579
3580         return ret;
3581 }
3582
3583 static int
3584 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3585                               uint32_t buffer_count,
3586                               struct drm_i915_gem_relocation_entry **relocs)
3587 {
3588         uint32_t reloc_count = 0, reloc_index = 0, i;
3589         int ret;
3590
3591         *relocs = NULL;
3592         for (i = 0; i < buffer_count; i++) {
3593                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3594                         return -EINVAL;
3595                 reloc_count += exec_list[i].relocation_count;
3596         }
3597
3598         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3599         if (*relocs == NULL) {
3600                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3601                 return -ENOMEM;
3602         }
3603
3604         for (i = 0; i < buffer_count; i++) {
3605                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3606
3607                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3608
3609                 ret = copy_from_user(&(*relocs)[reloc_index],
3610                                      user_relocs,
3611                                      exec_list[i].relocation_count *
3612                                      sizeof(**relocs));
3613                 if (ret != 0) {
3614                         drm_free_large(*relocs);
3615                         *relocs = NULL;
3616                         return -EFAULT;
3617                 }
3618
3619                 reloc_index += exec_list[i].relocation_count;
3620         }
3621
3622         return 0;
3623 }
3624
3625 static int
3626 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3627                             uint32_t buffer_count,
3628                             struct drm_i915_gem_relocation_entry *relocs)
3629 {
3630         uint32_t reloc_count = 0, i;
3631         int ret = 0;
3632
3633         if (relocs == NULL)
3634             return 0;
3635
3636         for (i = 0; i < buffer_count; i++) {
3637                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3638                 int unwritten;
3639
3640                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3641
3642                 unwritten = copy_to_user(user_relocs,
3643                                          &relocs[reloc_count],
3644                                          exec_list[i].relocation_count *
3645                                          sizeof(*relocs));
3646
3647                 if (unwritten) {
3648                         ret = -EFAULT;
3649                         goto err;
3650                 }
3651
3652                 reloc_count += exec_list[i].relocation_count;
3653         }
3654
3655 err:
3656         drm_free_large(relocs);
3657
3658         return ret;
3659 }
3660
3661 static int
3662 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3663                            uint64_t exec_offset)
3664 {
3665         uint32_t exec_start, exec_len;
3666
3667         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3668         exec_len = (uint32_t) exec->batch_len;
3669
3670         if ((exec_start | exec_len) & 0x7)
3671                 return -EINVAL;
3672
3673         if (!exec_start)
3674                 return -EINVAL;
3675
3676         return 0;
3677 }
3678
3679 static int
3680 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3681                                struct drm_gem_object **object_list,
3682                                int count)
3683 {
3684         drm_i915_private_t *dev_priv = dev->dev_private;
3685         struct drm_i915_gem_object *obj_priv;
3686         DEFINE_WAIT(wait);
3687         int i, ret = 0;
3688
3689         for (;;) {
3690                 prepare_to_wait(&dev_priv->pending_flip_queue,
3691                                 &wait, TASK_INTERRUPTIBLE);
3692                 for (i = 0; i < count; i++) {
3693                         obj_priv = object_list[i]->driver_private;
3694                         if (atomic_read(&obj_priv->pending_flip) > 0)
3695                                 break;
3696                 }
3697                 if (i == count)
3698                         break;
3699
3700                 if (!signal_pending(current)) {
3701                         mutex_unlock(&dev->struct_mutex);
3702                         schedule();
3703                         mutex_lock(&dev->struct_mutex);
3704                         continue;
3705                 }
3706                 ret = -ERESTARTSYS;
3707                 break;
3708         }
3709         finish_wait(&dev_priv->pending_flip_queue, &wait);
3710
3711         return ret;
3712 }
3713
3714 int
3715 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3716                        struct drm_file *file_priv,
3717                        struct drm_i915_gem_execbuffer2 *args,
3718                        struct drm_i915_gem_exec_object2 *exec_list)
3719 {
3720         drm_i915_private_t *dev_priv = dev->dev_private;
3721         struct drm_gem_object **object_list = NULL;
3722         struct drm_gem_object *batch_obj;
3723         struct drm_i915_gem_object *obj_priv;
3724         struct drm_clip_rect *cliprects = NULL;
3725         struct drm_i915_gem_relocation_entry *relocs = NULL;
3726         int ret = 0, ret2, i, pinned = 0;
3727         uint64_t exec_offset;
3728         uint32_t seqno, flush_domains, reloc_index;
3729         int pin_tries, flips;
3730
3731 #if WATCH_EXEC
3732         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3733                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3734 #endif
3735
3736         if (args->buffer_count < 1) {
3737                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3738                 return -EINVAL;
3739         }
3740         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3741         if (object_list == NULL) {
3742                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3743                           args->buffer_count);
3744                 ret = -ENOMEM;
3745                 goto pre_mutex_err;
3746         }
3747
3748         if (args->num_cliprects != 0) {
3749                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3750                                     GFP_KERNEL);
3751                 if (cliprects == NULL) {
3752                         ret = -ENOMEM;
3753                         goto pre_mutex_err;
3754                 }
3755
3756                 ret = copy_from_user(cliprects,
3757                                      (struct drm_clip_rect __user *)
3758                                      (uintptr_t) args->cliprects_ptr,
3759                                      sizeof(*cliprects) * args->num_cliprects);
3760                 if (ret != 0) {
3761                         DRM_ERROR("copy %d cliprects failed: %d\n",
3762                                   args->num_cliprects, ret);
3763                         goto pre_mutex_err;
3764                 }
3765         }
3766
3767         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3768                                             &relocs);
3769         if (ret != 0)
3770                 goto pre_mutex_err;
3771
3772         mutex_lock(&dev->struct_mutex);
3773
3774         i915_verify_inactive(dev, __FILE__, __LINE__);
3775
3776         if (atomic_read(&dev_priv->mm.wedged)) {
3777                 mutex_unlock(&dev->struct_mutex);
3778                 ret = -EIO;
3779                 goto pre_mutex_err;
3780         }
3781
3782         if (dev_priv->mm.suspended) {
3783                 mutex_unlock(&dev->struct_mutex);
3784                 ret = -EBUSY;
3785                 goto pre_mutex_err;
3786         }
3787
3788         /* Look up object handles */
3789         flips = 0;
3790         for (i = 0; i < args->buffer_count; i++) {
3791                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3792                                                        exec_list[i].handle);
3793                 if (object_list[i] == NULL) {
3794                         DRM_ERROR("Invalid object handle %d at index %d\n",
3795                                    exec_list[i].handle, i);
3796                         /* prevent error path from reading uninitialized data */
3797                         args->buffer_count = i + 1;
3798                         ret = -EBADF;
3799                         goto err;
3800                 }
3801
3802                 obj_priv = object_list[i]->driver_private;
3803                 if (obj_priv->in_execbuffer) {
3804                         DRM_ERROR("Object %p appears more than once in object list\n",
3805                                    object_list[i]);
3806                         /* prevent error path from reading uninitialized data */
3807                         args->buffer_count = i + 1;
3808                         ret = -EBADF;
3809                         goto err;
3810                 }
3811                 obj_priv->in_execbuffer = true;
3812                 flips += atomic_read(&obj_priv->pending_flip);
3813         }
3814
3815         if (flips > 0) {
3816                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3817                                                      args->buffer_count);
3818                 if (ret)
3819                         goto err;
3820         }
3821
3822         /* Pin and relocate */
3823         for (pin_tries = 0; ; pin_tries++) {
3824                 ret = 0;
3825                 reloc_index = 0;
3826
3827                 for (i = 0; i < args->buffer_count; i++) {
3828                         object_list[i]->pending_read_domains = 0;
3829                         object_list[i]->pending_write_domain = 0;
3830                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3831                                                                file_priv,
3832                                                                &exec_list[i],
3833                                                                &relocs[reloc_index]);
3834                         if (ret)
3835                                 break;
3836                         pinned = i + 1;
3837                         reloc_index += exec_list[i].relocation_count;
3838                 }
3839                 /* success */
3840                 if (ret == 0)
3841                         break;
3842
3843                 /* error other than GTT full, or we've already tried again */
3844                 if (ret != -ENOSPC || pin_tries >= 1) {
3845                         if (ret != -ERESTARTSYS) {
3846                                 unsigned long long total_size = 0;
3847                                 for (i = 0; i < args->buffer_count; i++)
3848                                         total_size += object_list[i]->size;
3849                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes: %d\n",
3850                                           pinned+1, args->buffer_count,
3851                                           total_size, ret);
3852                                 DRM_ERROR("%d objects [%d pinned], "
3853                                           "%d object bytes [%d pinned], "
3854                                           "%d/%d gtt bytes\n",
3855                                           atomic_read(&dev->object_count),
3856                                           atomic_read(&dev->pin_count),
3857                                           atomic_read(&dev->object_memory),
3858                                           atomic_read(&dev->pin_memory),
3859                                           atomic_read(&dev->gtt_memory),
3860                                           dev->gtt_total);
3861                         }
3862                         goto err;
3863                 }
3864
3865                 /* unpin all of our buffers */
3866                 for (i = 0; i < pinned; i++)
3867                         i915_gem_object_unpin(object_list[i]);
3868                 pinned = 0;
3869
3870                 /* evict everyone we can from the aperture */
3871                 ret = i915_gem_evict_everything(dev);
3872                 if (ret && ret != -ENOSPC)
3873                         goto err;
3874         }
3875
3876         /* Set the pending read domains for the batch buffer to COMMAND */
3877         batch_obj = object_list[args->buffer_count-1];
3878         if (batch_obj->pending_write_domain) {
3879                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3880                 ret = -EINVAL;
3881                 goto err;
3882         }
3883         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3884
3885         /* Sanity check the batch buffer, prior to moving objects */
3886         exec_offset = exec_list[args->buffer_count - 1].offset;
3887         ret = i915_gem_check_execbuffer (args, exec_offset);
3888         if (ret != 0) {
3889                 DRM_ERROR("execbuf with invalid offset/length\n");
3890                 goto err;
3891         }
3892
3893         i915_verify_inactive(dev, __FILE__, __LINE__);
3894
3895         /* Zero the global flush/invalidate flags. These
3896          * will be modified as new domains are computed
3897          * for each object
3898          */
3899         dev->invalidate_domains = 0;
3900         dev->flush_domains = 0;
3901
3902         for (i = 0; i < args->buffer_count; i++) {
3903                 struct drm_gem_object *obj = object_list[i];
3904
3905                 /* Compute new gpu domains and update invalidate/flush */
3906                 i915_gem_object_set_to_gpu_domain(obj);
3907         }
3908
3909         i915_verify_inactive(dev, __FILE__, __LINE__);
3910
3911         if (dev->invalidate_domains | dev->flush_domains) {
3912 #if WATCH_EXEC
3913                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3914                           __func__,
3915                          dev->invalidate_domains,
3916                          dev->flush_domains);
3917 #endif
3918                 i915_gem_flush(dev,
3919                                dev->invalidate_domains,
3920                                dev->flush_domains);
3921                 if (dev->flush_domains & I915_GEM_GPU_DOMAINS)
3922                         (void)i915_add_request(dev, file_priv,
3923                                                dev->flush_domains);
3924         }
3925
3926         for (i = 0; i < args->buffer_count; i++) {
3927                 struct drm_gem_object *obj = object_list[i];
3928                 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3929                 uint32_t old_write_domain = obj->write_domain;
3930
3931                 obj->write_domain = obj->pending_write_domain;
3932                 if (obj->write_domain)
3933                         list_move_tail(&obj_priv->gpu_write_list,
3934                                        &dev_priv->mm.gpu_write_list);
3935                 else
3936                         list_del_init(&obj_priv->gpu_write_list);
3937
3938                 trace_i915_gem_object_change_domain(obj,
3939                                                     obj->read_domains,
3940                                                     old_write_domain);
3941         }
3942
3943         i915_verify_inactive(dev, __FILE__, __LINE__);
3944
3945 #if WATCH_COHERENCY
3946         for (i = 0; i < args->buffer_count; i++) {
3947                 i915_gem_object_check_coherency(object_list[i],
3948                                                 exec_list[i].handle);
3949         }
3950 #endif
3951
3952 #if WATCH_EXEC
3953         i915_gem_dump_object(batch_obj,
3954                               args->batch_len,
3955                               __func__,
3956                               ~0);
3957 #endif
3958
3959         /* Exec the batchbuffer */
3960         ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
3961         if (ret) {
3962                 DRM_ERROR("dispatch failed %d\n", ret);
3963                 goto err;
3964         }
3965
3966         /*
3967          * Ensure that the commands in the batch buffer are
3968          * finished before the interrupt fires
3969          */
3970         flush_domains = i915_retire_commands(dev);
3971
3972         i915_verify_inactive(dev, __FILE__, __LINE__);
3973
3974         /*
3975          * Get a seqno representing the execution of the current buffer,
3976          * which we can wait on.  We would like to mitigate these interrupts,
3977          * likely by only creating seqnos occasionally (so that we have
3978          * *some* interrupts representing completion of buffers that we can
3979          * wait on when trying to clear up gtt space).
3980          */
3981         seqno = i915_add_request(dev, file_priv, flush_domains);
3982         BUG_ON(seqno == 0);
3983         for (i = 0; i < args->buffer_count; i++) {
3984                 struct drm_gem_object *obj = object_list[i];
3985
3986                 i915_gem_object_move_to_active(obj, seqno);
3987 #if WATCH_LRU
3988                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3989 #endif
3990         }
3991 #if WATCH_LRU
3992         i915_dump_lru(dev, __func__);
3993 #endif
3994
3995         i915_verify_inactive(dev, __FILE__, __LINE__);
3996
3997 err:
3998         for (i = 0; i < pinned; i++)
3999                 i915_gem_object_unpin(object_list[i]);
4000
4001         for (i = 0; i < args->buffer_count; i++) {
4002                 if (object_list[i]) {
4003                         obj_priv = object_list[i]->driver_private;
4004                         obj_priv->in_execbuffer = false;
4005                 }
4006                 drm_gem_object_unreference(object_list[i]);
4007         }
4008
4009         mutex_unlock(&dev->struct_mutex);
4010
4011 pre_mutex_err:
4012         /* Copy the updated relocations out regardless of current error
4013          * state.  Failure to update the relocs would mean that the next
4014          * time userland calls execbuf, it would do so with presumed offset
4015          * state that didn't match the actual object state.
4016          */
4017         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
4018                                            relocs);
4019         if (ret2 != 0) {
4020                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
4021
4022                 if (ret == 0)
4023                         ret = ret2;
4024         }
4025
4026         drm_free_large(object_list);
4027         kfree(cliprects);
4028
4029         return ret;
4030 }
4031
4032 /*
4033  * Legacy execbuffer just creates an exec2 list from the original exec object
4034  * list array and passes it to the real function.
4035  */
4036 int
4037 i915_gem_execbuffer(struct drm_device *dev, void *data,
4038                     struct drm_file *file_priv)
4039 {
4040         struct drm_i915_gem_execbuffer *args = data;
4041         struct drm_i915_gem_execbuffer2 exec2;
4042         struct drm_i915_gem_exec_object *exec_list = NULL;
4043         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4044         int ret, i;
4045
4046 #if WATCH_EXEC
4047         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4048                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4049 #endif
4050
4051         if (args->buffer_count < 1) {
4052                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4053                 return -EINVAL;
4054         }
4055
4056         /* Copy in the exec list from userland */
4057         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4058         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4059         if (exec_list == NULL || exec2_list == NULL) {
4060                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4061                           args->buffer_count);
4062                 drm_free_large(exec_list);
4063                 drm_free_large(exec2_list);
4064                 return -ENOMEM;
4065         }
4066         ret = copy_from_user(exec_list,
4067                              (struct drm_i915_relocation_entry __user *)
4068                              (uintptr_t) args->buffers_ptr,
4069                              sizeof(*exec_list) * args->buffer_count);
4070         if (ret != 0) {
4071                 DRM_ERROR("copy %d exec entries failed %d\n",
4072                           args->buffer_count, ret);
4073                 drm_free_large(exec_list);
4074                 drm_free_large(exec2_list);
4075                 return -EFAULT;
4076         }
4077
4078         for (i = 0; i < args->buffer_count; i++) {
4079                 exec2_list[i].handle = exec_list[i].handle;
4080                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4081                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4082                 exec2_list[i].alignment = exec_list[i].alignment;
4083                 exec2_list[i].offset = exec_list[i].offset;
4084                 if (!IS_I965G(dev))
4085                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4086                 else
4087                         exec2_list[i].flags = 0;
4088         }
4089
4090         exec2.buffers_ptr = args->buffers_ptr;
4091         exec2.buffer_count = args->buffer_count;
4092         exec2.batch_start_offset = args->batch_start_offset;
4093         exec2.batch_len = args->batch_len;
4094         exec2.DR1 = args->DR1;
4095         exec2.DR4 = args->DR4;
4096         exec2.num_cliprects = args->num_cliprects;
4097         exec2.cliprects_ptr = args->cliprects_ptr;
4098         exec2.flags = 0;
4099
4100         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4101         if (!ret) {
4102                 /* Copy the new buffer offsets back to the user's exec list. */
4103                 for (i = 0; i < args->buffer_count; i++)
4104                         exec_list[i].offset = exec2_list[i].offset;
4105                 /* ... and back out to userspace */
4106                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4107                                    (uintptr_t) args->buffers_ptr,
4108                                    exec_list,
4109                                    sizeof(*exec_list) * args->buffer_count);
4110                 if (ret) {
4111                         ret = -EFAULT;
4112                         DRM_ERROR("failed to copy %d exec entries "
4113                                   "back to user (%d)\n",
4114                                   args->buffer_count, ret);
4115                 }
4116         }
4117
4118         drm_free_large(exec_list);
4119         drm_free_large(exec2_list);
4120         return ret;
4121 }
4122
4123 int
4124 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4125                      struct drm_file *file_priv)
4126 {
4127         struct drm_i915_gem_execbuffer2 *args = data;
4128         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4129         int ret;
4130
4131 #if WATCH_EXEC
4132         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4133                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4134 #endif
4135
4136         if (args->buffer_count < 1) {
4137                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4138                 return -EINVAL;
4139         }
4140
4141         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4142         if (exec2_list == NULL) {
4143                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4144                           args->buffer_count);
4145                 return -ENOMEM;
4146         }
4147         ret = copy_from_user(exec2_list,
4148                              (struct drm_i915_relocation_entry __user *)
4149                              (uintptr_t) args->buffers_ptr,
4150                              sizeof(*exec2_list) * args->buffer_count);
4151         if (ret != 0) {
4152                 DRM_ERROR("copy %d exec entries failed %d\n",
4153                           args->buffer_count, ret);
4154                 drm_free_large(exec2_list);
4155                 return -EFAULT;
4156         }
4157
4158         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4159         if (!ret) {
4160                 /* Copy the new buffer offsets back to the user's exec list. */
4161                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4162                                    (uintptr_t) args->buffers_ptr,
4163                                    exec2_list,
4164                                    sizeof(*exec2_list) * args->buffer_count);
4165                 if (ret) {
4166                         ret = -EFAULT;
4167                         DRM_ERROR("failed to copy %d exec entries "
4168                                   "back to user (%d)\n",
4169                                   args->buffer_count, ret);
4170                 }
4171         }
4172
4173         drm_free_large(exec2_list);
4174         return ret;
4175 }
4176
4177 int
4178 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4179 {
4180         struct drm_device *dev = obj->dev;
4181         struct drm_i915_gem_object *obj_priv = obj->driver_private;
4182         int ret;
4183
4184         i915_verify_inactive(dev, __FILE__, __LINE__);
4185         if (obj_priv->gtt_space == NULL) {
4186                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4187                 if (ret)
4188                         return ret;
4189         }
4190
4191         obj_priv->pin_count++;
4192
4193         /* If the object is not active and not pending a flush,
4194          * remove it from the inactive list
4195          */
4196         if (obj_priv->pin_count == 1) {
4197                 atomic_inc(&dev->pin_count);
4198                 atomic_add(obj->size, &dev->pin_memory);
4199                 if (!obj_priv->active &&
4200                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
4201                     !list_empty(&obj_priv->list))
4202                         list_del_init(&obj_priv->list);
4203         }
4204         i915_verify_inactive(dev, __FILE__, __LINE__);
4205
4206         return 0;
4207 }
4208
4209 void
4210 i915_gem_object_unpin(struct drm_gem_object *obj)
4211 {
4212         struct drm_device *dev = obj->dev;
4213         drm_i915_private_t *dev_priv = dev->dev_private;
4214         struct drm_i915_gem_object *obj_priv = obj->driver_private;
4215
4216         i915_verify_inactive(dev, __FILE__, __LINE__);
4217         obj_priv->pin_count--;
4218         BUG_ON(obj_priv->pin_count < 0);
4219         BUG_ON(obj_priv->gtt_space == NULL);
4220
4221         /* If the object is no longer pinned, and is
4222          * neither active nor being flushed, then stick it on
4223          * the inactive list
4224          */
4225         if (obj_priv->pin_count == 0) {
4226                 if (!obj_priv->active &&
4227                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4228                         list_move_tail(&obj_priv->list,
4229                                        &dev_priv->mm.inactive_list);
4230                 atomic_dec(&dev->pin_count);
4231                 atomic_sub(obj->size, &dev->pin_memory);
4232         }
4233         i915_verify_inactive(dev, __FILE__, __LINE__);
4234 }
4235
4236 int
4237 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4238                    struct drm_file *file_priv)
4239 {
4240         struct drm_i915_gem_pin *args = data;
4241         struct drm_gem_object *obj;
4242         struct drm_i915_gem_object *obj_priv;
4243         int ret;
4244
4245         mutex_lock(&dev->struct_mutex);
4246
4247         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4248         if (obj == NULL) {
4249                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4250                           args->handle);
4251                 mutex_unlock(&dev->struct_mutex);
4252                 return -EBADF;
4253         }
4254         obj_priv = obj->driver_private;
4255
4256         if (obj_priv->madv != I915_MADV_WILLNEED) {
4257                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4258                 drm_gem_object_unreference(obj);
4259                 mutex_unlock(&dev->struct_mutex);
4260                 return -EINVAL;
4261         }
4262
4263         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4264                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4265                           args->handle);
4266                 drm_gem_object_unreference(obj);
4267                 mutex_unlock(&dev->struct_mutex);
4268                 return -EINVAL;
4269         }
4270
4271         obj_priv->user_pin_count++;
4272         obj_priv->pin_filp = file_priv;
4273         if (obj_priv->user_pin_count == 1) {
4274                 ret = i915_gem_object_pin(obj, args->alignment);
4275                 if (ret != 0) {
4276                         drm_gem_object_unreference(obj);
4277                         mutex_unlock(&dev->struct_mutex);
4278                         return ret;
4279                 }
4280         }
4281
4282         /* XXX - flush the CPU caches for pinned objects
4283          * as the X server doesn't manage domains yet
4284          */
4285         i915_gem_object_flush_cpu_write_domain(obj);
4286         args->offset = obj_priv->gtt_offset;
4287         drm_gem_object_unreference(obj);
4288         mutex_unlock(&dev->struct_mutex);
4289
4290         return 0;
4291 }
4292
4293 int
4294 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4295                      struct drm_file *file_priv)
4296 {
4297         struct drm_i915_gem_pin *args = data;
4298         struct drm_gem_object *obj;
4299         struct drm_i915_gem_object *obj_priv;
4300
4301         mutex_lock(&dev->struct_mutex);
4302
4303         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4304         if (obj == NULL) {
4305                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4306                           args->handle);
4307                 mutex_unlock(&dev->struct_mutex);
4308                 return -EBADF;
4309         }
4310
4311         obj_priv = obj->driver_private;
4312         if (obj_priv->pin_filp != file_priv) {
4313                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4314                           args->handle);
4315                 drm_gem_object_unreference(obj);
4316                 mutex_unlock(&dev->struct_mutex);
4317                 return -EINVAL;
4318         }
4319         obj_priv->user_pin_count--;
4320         if (obj_priv->user_pin_count == 0) {
4321                 obj_priv->pin_filp = NULL;
4322                 i915_gem_object_unpin(obj);
4323         }
4324
4325         drm_gem_object_unreference(obj);
4326         mutex_unlock(&dev->struct_mutex);
4327         return 0;
4328 }
4329
4330 int
4331 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4332                     struct drm_file *file_priv)
4333 {
4334         struct drm_i915_gem_busy *args = data;
4335         struct drm_gem_object *obj;
4336         struct drm_i915_gem_object *obj_priv;
4337
4338         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4339         if (obj == NULL) {
4340                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4341                           args->handle);
4342                 return -EBADF;
4343         }
4344
4345         mutex_lock(&dev->struct_mutex);
4346         /* Update the active list for the hardware's current position.
4347          * Otherwise this only updates on a delayed timer or when irqs are
4348          * actually unmasked, and our working set ends up being larger than
4349          * required.
4350          */
4351         i915_gem_retire_requests(dev);
4352
4353         obj_priv = obj->driver_private;
4354         /* Don't count being on the flushing list against the object being
4355          * done.  Otherwise, a buffer left on the flushing list but not getting
4356          * flushed (because nobody's flushing that domain) won't ever return
4357          * unbusy and get reused by libdrm's bo cache.  The other expected
4358          * consumer of this interface, OpenGL's occlusion queries, also specs
4359          * that the objects get unbusy "eventually" without any interference.
4360          */
4361         args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
4362
4363         drm_gem_object_unreference(obj);
4364         mutex_unlock(&dev->struct_mutex);
4365         return 0;
4366 }
4367
4368 int
4369 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4370                         struct drm_file *file_priv)
4371 {
4372     return i915_gem_ring_throttle(dev, file_priv);
4373 }
4374
4375 int
4376 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4377                        struct drm_file *file_priv)
4378 {
4379         struct drm_i915_gem_madvise *args = data;
4380         struct drm_gem_object *obj;
4381         struct drm_i915_gem_object *obj_priv;
4382
4383         switch (args->madv) {
4384         case I915_MADV_DONTNEED:
4385         case I915_MADV_WILLNEED:
4386             break;
4387         default:
4388             return -EINVAL;
4389         }
4390
4391         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4392         if (obj == NULL) {
4393                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4394                           args->handle);
4395                 return -EBADF;
4396         }
4397
4398         mutex_lock(&dev->struct_mutex);
4399         obj_priv = obj->driver_private;
4400
4401         if (obj_priv->pin_count) {
4402                 drm_gem_object_unreference(obj);
4403                 mutex_unlock(&dev->struct_mutex);
4404
4405                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4406                 return -EINVAL;
4407         }
4408
4409         if (obj_priv->madv != __I915_MADV_PURGED)
4410                 obj_priv->madv = args->madv;
4411
4412         /* if the object is no longer bound, discard its backing storage */
4413         if (i915_gem_object_is_purgeable(obj_priv) &&
4414             obj_priv->gtt_space == NULL)
4415                 i915_gem_object_truncate(obj);
4416
4417         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4418
4419         drm_gem_object_unreference(obj);
4420         mutex_unlock(&dev->struct_mutex);
4421
4422         return 0;
4423 }
4424
4425 int i915_gem_init_object(struct drm_gem_object *obj)
4426 {
4427         struct drm_i915_gem_object *obj_priv;
4428
4429         obj_priv = kzalloc(sizeof(*obj_priv), GFP_KERNEL);
4430         if (obj_priv == NULL)
4431                 return -ENOMEM;
4432
4433         /*
4434          * We've just allocated pages from the kernel,
4435          * so they've just been written by the CPU with
4436          * zeros. They'll need to be clflushed before we
4437          * use them with the GPU.
4438          */
4439         obj->write_domain = I915_GEM_DOMAIN_CPU;
4440         obj->read_domains = I915_GEM_DOMAIN_CPU;
4441
4442         obj_priv->agp_type = AGP_USER_MEMORY;
4443
4444         obj->driver_private = obj_priv;
4445         obj_priv->obj = obj;
4446         obj_priv->fence_reg = I915_FENCE_REG_NONE;
4447         INIT_LIST_HEAD(&obj_priv->list);
4448         INIT_LIST_HEAD(&obj_priv->gpu_write_list);
4449         INIT_LIST_HEAD(&obj_priv->fence_list);
4450         obj_priv->madv = I915_MADV_WILLNEED;
4451
4452         trace_i915_gem_object_create(obj);
4453
4454         return 0;
4455 }
4456
4457 void i915_gem_free_object(struct drm_gem_object *obj)
4458 {
4459         struct drm_device *dev = obj->dev;
4460         struct drm_i915_gem_object *obj_priv = obj->driver_private;
4461
4462         trace_i915_gem_object_destroy(obj);
4463
4464         while (obj_priv->pin_count > 0)
4465                 i915_gem_object_unpin(obj);
4466
4467         if (obj_priv->phys_obj)
4468                 i915_gem_detach_phys_object(dev, obj);
4469
4470         i915_gem_object_unbind(obj);
4471
4472         if (obj_priv->mmap_offset)
4473                 i915_gem_free_mmap_offset(obj);
4474
4475         kfree(obj_priv->page_cpu_valid);
4476         kfree(obj_priv->bit_17);
4477         kfree(obj->driver_private);
4478 }
4479
4480 /** Unbinds all inactive objects. */
4481 static int
4482 i915_gem_evict_from_inactive_list(struct drm_device *dev)
4483 {
4484         drm_i915_private_t *dev_priv = dev->dev_private;
4485
4486         while (!list_empty(&dev_priv->mm.inactive_list)) {
4487                 struct drm_gem_object *obj;
4488                 int ret;
4489
4490                 obj = list_first_entry(&dev_priv->mm.inactive_list,
4491                                        struct drm_i915_gem_object,
4492                                        list)->obj;
4493
4494                 ret = i915_gem_object_unbind(obj);
4495                 if (ret != 0) {
4496                         DRM_ERROR("Error unbinding object: %d\n", ret);
4497                         return ret;
4498                 }
4499         }
4500
4501         return 0;
4502 }
4503
4504 int
4505 i915_gem_idle(struct drm_device *dev)
4506 {
4507         drm_i915_private_t *dev_priv = dev->dev_private;
4508         int ret;
4509
4510         mutex_lock(&dev->struct_mutex);
4511
4512         if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
4513                 mutex_unlock(&dev->struct_mutex);
4514                 return 0;
4515         }
4516
4517         ret = i915_gpu_idle(dev);
4518         if (ret) {
4519                 mutex_unlock(&dev->struct_mutex);
4520                 return ret;
4521         }
4522
4523         /* Under UMS, be paranoid and evict. */
4524         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4525                 ret = i915_gem_evict_from_inactive_list(dev);
4526                 if (ret) {
4527                         mutex_unlock(&dev->struct_mutex);
4528                         return ret;
4529                 }
4530         }
4531
4532         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4533          * We need to replace this with a semaphore, or something.
4534          * And not confound mm.suspended!
4535          */
4536         dev_priv->mm.suspended = 1;
4537         del_timer(&dev_priv->hangcheck_timer);
4538
4539         i915_kernel_lost_context(dev);
4540         i915_gem_cleanup_ringbuffer(dev);
4541
4542         mutex_unlock(&dev->struct_mutex);
4543
4544         /* Cancel the retire work handler, which should be idle now. */
4545         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4546
4547         return 0;
4548 }
4549
4550 static int
4551 i915_gem_init_hws(struct drm_device *dev)
4552 {
4553         drm_i915_private_t *dev_priv = dev->dev_private;
4554         struct drm_gem_object *obj;
4555         struct drm_i915_gem_object *obj_priv;
4556         int ret;
4557
4558         /* If we need a physical address for the status page, it's already
4559          * initialized at driver load time.
4560          */
4561         if (!I915_NEED_GFX_HWS(dev))
4562                 return 0;
4563
4564         obj = drm_gem_object_alloc(dev, 4096);
4565         if (obj == NULL) {
4566                 DRM_ERROR("Failed to allocate status page\n");
4567                 return -ENOMEM;
4568         }
4569         obj_priv = obj->driver_private;
4570         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4571
4572         ret = i915_gem_object_pin(obj, 4096);
4573         if (ret != 0) {
4574                 drm_gem_object_unreference(obj);
4575                 return ret;
4576         }
4577
4578         dev_priv->status_gfx_addr = obj_priv->gtt_offset;
4579
4580         dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
4581         if (dev_priv->hw_status_page == NULL) {
4582                 DRM_ERROR("Failed to map status page.\n");
4583                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4584                 i915_gem_object_unpin(obj);
4585                 drm_gem_object_unreference(obj);
4586                 return -EINVAL;
4587         }
4588         dev_priv->hws_obj = obj;
4589         memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
4590         if (IS_GEN6(dev)) {
4591                 I915_WRITE(HWS_PGA_GEN6, dev_priv->status_gfx_addr);
4592                 I915_READ(HWS_PGA_GEN6); /* posting read */
4593         } else {
4594                 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
4595                 I915_READ(HWS_PGA); /* posting read */
4596         }
4597         DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
4598
4599         return 0;
4600 }
4601
4602 static void
4603 i915_gem_cleanup_hws(struct drm_device *dev)
4604 {
4605         drm_i915_private_t *dev_priv = dev->dev_private;
4606         struct drm_gem_object *obj;
4607         struct drm_i915_gem_object *obj_priv;
4608
4609         if (dev_priv->hws_obj == NULL)
4610                 return;
4611
4612         obj = dev_priv->hws_obj;
4613         obj_priv = obj->driver_private;
4614
4615         kunmap(obj_priv->pages[0]);
4616         i915_gem_object_unpin(obj);
4617         drm_gem_object_unreference(obj);
4618         dev_priv->hws_obj = NULL;
4619
4620         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4621         dev_priv->hw_status_page = NULL;
4622
4623         /* Write high address into HWS_PGA when disabling. */
4624         I915_WRITE(HWS_PGA, 0x1ffff000);
4625 }
4626
4627 int
4628 i915_gem_init_ringbuffer(struct drm_device *dev)
4629 {
4630         drm_i915_private_t *dev_priv = dev->dev_private;
4631         struct drm_gem_object *obj;
4632         struct drm_i915_gem_object *obj_priv;
4633         drm_i915_ring_buffer_t *ring = &dev_priv->ring;
4634         int ret;
4635         u32 head;
4636
4637         ret = i915_gem_init_hws(dev);
4638         if (ret != 0)
4639                 return ret;
4640
4641         obj = drm_gem_object_alloc(dev, 128 * 1024);
4642         if (obj == NULL) {
4643                 DRM_ERROR("Failed to allocate ringbuffer\n");
4644                 i915_gem_cleanup_hws(dev);
4645                 return -ENOMEM;
4646         }
4647         obj_priv = obj->driver_private;
4648
4649         ret = i915_gem_object_pin(obj, 4096);
4650         if (ret != 0) {
4651                 drm_gem_object_unreference(obj);
4652                 i915_gem_cleanup_hws(dev);
4653                 return ret;
4654         }
4655
4656         /* Set up the kernel mapping for the ring. */
4657         ring->Size = obj->size;
4658
4659         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4660         ring->map.size = obj->size;
4661         ring->map.type = 0;
4662         ring->map.flags = 0;
4663         ring->map.mtrr = 0;
4664
4665         drm_core_ioremap_wc(&ring->map, dev);
4666         if (ring->map.handle == NULL) {
4667                 DRM_ERROR("Failed to map ringbuffer.\n");
4668                 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4669                 i915_gem_object_unpin(obj);
4670                 drm_gem_object_unreference(obj);
4671                 i915_gem_cleanup_hws(dev);
4672                 return -EINVAL;
4673         }
4674         ring->ring_obj = obj;
4675         ring->virtual_start = ring->map.handle;
4676
4677         /* Stop the ring if it's running. */
4678         I915_WRITE(PRB0_CTL, 0);
4679         I915_WRITE(PRB0_TAIL, 0);
4680         I915_WRITE(PRB0_HEAD, 0);
4681
4682         /* Initialize the ring. */
4683         I915_WRITE(PRB0_START, obj_priv->gtt_offset);
4684         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4685
4686         /* G45 ring initialization fails to reset head to zero */
4687         if (head != 0) {
4688                 DRM_ERROR("Ring head not reset to zero "
4689                           "ctl %08x head %08x tail %08x start %08x\n",
4690                           I915_READ(PRB0_CTL),
4691                           I915_READ(PRB0_HEAD),
4692                           I915_READ(PRB0_TAIL),
4693                           I915_READ(PRB0_START));
4694                 I915_WRITE(PRB0_HEAD, 0);
4695
4696                 DRM_ERROR("Ring head forced to zero "
4697                           "ctl %08x head %08x tail %08x start %08x\n",
4698                           I915_READ(PRB0_CTL),
4699                           I915_READ(PRB0_HEAD),
4700                           I915_READ(PRB0_TAIL),
4701                           I915_READ(PRB0_START));
4702         }
4703
4704         I915_WRITE(PRB0_CTL,
4705                    ((obj->size - 4096) & RING_NR_PAGES) |
4706                    RING_NO_REPORT |
4707                    RING_VALID);
4708
4709         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4710
4711         /* If the head is still not zero, the ring is dead */
4712         if (head != 0) {
4713                 DRM_ERROR("Ring initialization failed "
4714                           "ctl %08x head %08x tail %08x start %08x\n",
4715                           I915_READ(PRB0_CTL),
4716                           I915_READ(PRB0_HEAD),
4717                           I915_READ(PRB0_TAIL),
4718                           I915_READ(PRB0_START));
4719                 return -EIO;
4720         }
4721
4722         /* Update our cache of the ring state */
4723         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4724                 i915_kernel_lost_context(dev);
4725         else {
4726                 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4727                 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4728                 ring->space = ring->head - (ring->tail + 8);
4729                 if (ring->space < 0)
4730                         ring->space += ring->Size;
4731         }
4732
4733         return 0;
4734 }
4735
4736 void
4737 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4738 {
4739         drm_i915_private_t *dev_priv = dev->dev_private;
4740
4741         if (dev_priv->ring.ring_obj == NULL)
4742                 return;
4743
4744         drm_core_ioremapfree(&dev_priv->ring.map, dev);
4745
4746         i915_gem_object_unpin(dev_priv->ring.ring_obj);
4747         drm_gem_object_unreference(dev_priv->ring.ring_obj);
4748         dev_priv->ring.ring_obj = NULL;
4749         memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4750
4751         i915_gem_cleanup_hws(dev);
4752 }
4753
4754 int
4755 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4756                        struct drm_file *file_priv)
4757 {
4758         drm_i915_private_t *dev_priv = dev->dev_private;
4759         int ret;
4760
4761         if (drm_core_check_feature(dev, DRIVER_MODESET))
4762                 return 0;
4763
4764         if (atomic_read(&dev_priv->mm.wedged)) {
4765                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4766                 atomic_set(&dev_priv->mm.wedged, 0);
4767         }
4768
4769         mutex_lock(&dev->struct_mutex);
4770         dev_priv->mm.suspended = 0;
4771
4772         ret = i915_gem_init_ringbuffer(dev);
4773         if (ret != 0) {
4774                 mutex_unlock(&dev->struct_mutex);
4775                 return ret;
4776         }
4777
4778         spin_lock(&dev_priv->mm.active_list_lock);
4779         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4780         spin_unlock(&dev_priv->mm.active_list_lock);
4781
4782         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4783         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4784         BUG_ON(!list_empty(&dev_priv->mm.request_list));
4785         mutex_unlock(&dev->struct_mutex);
4786
4787         drm_irq_install(dev);
4788
4789         return 0;
4790 }
4791
4792 int
4793 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4794                        struct drm_file *file_priv)
4795 {
4796         if (drm_core_check_feature(dev, DRIVER_MODESET))
4797                 return 0;
4798
4799         drm_irq_uninstall(dev);
4800         return i915_gem_idle(dev);
4801 }
4802
4803 void
4804 i915_gem_lastclose(struct drm_device *dev)
4805 {
4806         int ret;
4807
4808         if (drm_core_check_feature(dev, DRIVER_MODESET))
4809                 return;
4810
4811         ret = i915_gem_idle(dev);
4812         if (ret)
4813                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4814 }
4815
4816 void
4817 i915_gem_load(struct drm_device *dev)
4818 {
4819         int i;
4820         drm_i915_private_t *dev_priv = dev->dev_private;
4821
4822         spin_lock_init(&dev_priv->mm.active_list_lock);
4823         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4824         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4825         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4826         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4827         INIT_LIST_HEAD(&dev_priv->mm.request_list);
4828         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4829         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4830                           i915_gem_retire_work_handler);
4831         dev_priv->mm.next_gem_seqno = 1;
4832
4833         spin_lock(&shrink_list_lock);
4834         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4835         spin_unlock(&shrink_list_lock);
4836
4837         /* Old X drivers will take 0-2 for front, back, depth buffers */
4838         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4839                 dev_priv->fence_reg_start = 3;
4840
4841         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4842                 dev_priv->num_fence_regs = 16;
4843         else
4844                 dev_priv->num_fence_regs = 8;
4845
4846         /* Initialize fence registers to zero */
4847         if (IS_I965G(dev)) {
4848                 for (i = 0; i < 16; i++)
4849                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4850         } else {
4851                 for (i = 0; i < 8; i++)
4852                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4853                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4854                         for (i = 0; i < 8; i++)
4855                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4856         }
4857         i915_gem_detect_bit_6_swizzle(dev);
4858         init_waitqueue_head(&dev_priv->pending_flip_queue);
4859 }
4860
4861 /*
4862  * Create a physically contiguous memory object for this object
4863  * e.g. for cursor + overlay regs
4864  */
4865 int i915_gem_init_phys_object(struct drm_device *dev,
4866                               int id, int size)
4867 {
4868         drm_i915_private_t *dev_priv = dev->dev_private;
4869         struct drm_i915_gem_phys_object *phys_obj;
4870         int ret;
4871
4872         if (dev_priv->mm.phys_objs[id - 1] || !size)
4873                 return 0;
4874
4875         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4876         if (!phys_obj)
4877                 return -ENOMEM;
4878
4879         phys_obj->id = id;
4880
4881         phys_obj->handle = drm_pci_alloc(dev, size, 0);
4882         if (!phys_obj->handle) {
4883                 ret = -ENOMEM;
4884                 goto kfree_obj;
4885         }
4886 #ifdef CONFIG_X86
4887         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4888 #endif
4889
4890         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4891
4892         return 0;
4893 kfree_obj:
4894         kfree(phys_obj);
4895         return ret;
4896 }
4897
4898 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4899 {
4900         drm_i915_private_t *dev_priv = dev->dev_private;
4901         struct drm_i915_gem_phys_object *phys_obj;
4902
4903         if (!dev_priv->mm.phys_objs[id - 1])
4904                 return;
4905
4906         phys_obj = dev_priv->mm.phys_objs[id - 1];
4907         if (phys_obj->cur_obj) {
4908                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4909         }
4910
4911 #ifdef CONFIG_X86
4912         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4913 #endif
4914         drm_pci_free(dev, phys_obj->handle);
4915         kfree(phys_obj);
4916         dev_priv->mm.phys_objs[id - 1] = NULL;
4917 }
4918
4919 void i915_gem_free_all_phys_object(struct drm_device *dev)
4920 {
4921         int i;
4922
4923         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4924                 i915_gem_free_phys_object(dev, i);
4925 }
4926
4927 void i915_gem_detach_phys_object(struct drm_device *dev,
4928                                  struct drm_gem_object *obj)
4929 {
4930         struct drm_i915_gem_object *obj_priv;
4931         int i;
4932         int ret;
4933         int page_count;
4934
4935         obj_priv = obj->driver_private;
4936         if (!obj_priv->phys_obj)
4937                 return;
4938
4939         ret = i915_gem_object_get_pages(obj, 0);
4940         if (ret)
4941                 goto out;
4942
4943         page_count = obj->size / PAGE_SIZE;
4944
4945         for (i = 0; i < page_count; i++) {
4946                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4947                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4948
4949                 memcpy(dst, src, PAGE_SIZE);
4950                 kunmap_atomic(dst, KM_USER0);
4951         }
4952         drm_clflush_pages(obj_priv->pages, page_count);
4953         drm_agp_chipset_flush(dev);
4954
4955         i915_gem_object_put_pages(obj);
4956 out:
4957         obj_priv->phys_obj->cur_obj = NULL;
4958         obj_priv->phys_obj = NULL;
4959 }
4960
4961 int
4962 i915_gem_attach_phys_object(struct drm_device *dev,
4963                             struct drm_gem_object *obj, int id)
4964 {
4965         drm_i915_private_t *dev_priv = dev->dev_private;
4966         struct drm_i915_gem_object *obj_priv;
4967         int ret = 0;
4968         int page_count;
4969         int i;
4970
4971         if (id > I915_MAX_PHYS_OBJECT)
4972                 return -EINVAL;
4973
4974         obj_priv = obj->driver_private;
4975
4976         if (obj_priv->phys_obj) {
4977                 if (obj_priv->phys_obj->id == id)
4978                         return 0;
4979                 i915_gem_detach_phys_object(dev, obj);
4980         }
4981
4982
4983         /* create a new object */
4984         if (!dev_priv->mm.phys_objs[id - 1]) {
4985                 ret = i915_gem_init_phys_object(dev, id,
4986                                                 obj->size);
4987                 if (ret) {
4988                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4989                         goto out;
4990                 }
4991         }
4992
4993         /* bind to the object */
4994         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4995         obj_priv->phys_obj->cur_obj = obj;
4996
4997         ret = i915_gem_object_get_pages(obj, 0);
4998         if (ret) {
4999                 DRM_ERROR("failed to get page list\n");
5000                 goto out;
5001         }
5002
5003         page_count = obj->size / PAGE_SIZE;
5004
5005         for (i = 0; i < page_count; i++) {
5006                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
5007                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5008
5009                 memcpy(dst, src, PAGE_SIZE);
5010                 kunmap_atomic(src, KM_USER0);
5011         }
5012
5013         i915_gem_object_put_pages(obj);
5014
5015         return 0;
5016 out:
5017         return ret;
5018 }
5019
5020 static int
5021 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5022                      struct drm_i915_gem_pwrite *args,
5023                      struct drm_file *file_priv)
5024 {
5025         struct drm_i915_gem_object *obj_priv = obj->driver_private;
5026         void *obj_addr;
5027         int ret;
5028         char __user *user_data;
5029
5030         user_data = (char __user *) (uintptr_t) args->data_ptr;
5031         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
5032
5033         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
5034         ret = copy_from_user(obj_addr, user_data, args->size);
5035         if (ret)
5036                 return -EFAULT;
5037
5038         drm_agp_chipset_flush(dev);
5039         return 0;
5040 }
5041
5042 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
5043 {
5044         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
5045
5046         /* Clean up our request list when the client is going away, so that
5047          * later retire_requests won't dereference our soon-to-be-gone
5048          * file_priv.
5049          */
5050         mutex_lock(&dev->struct_mutex);
5051         while (!list_empty(&i915_file_priv->mm.request_list))
5052                 list_del_init(i915_file_priv->mm.request_list.next);
5053         mutex_unlock(&dev->struct_mutex);
5054 }
5055
5056 static int
5057 i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
5058 {
5059         drm_i915_private_t *dev_priv, *next_dev;
5060         struct drm_i915_gem_object *obj_priv, *next_obj;
5061         int cnt = 0;
5062         int would_deadlock = 1;
5063
5064         /* "fast-path" to count number of available objects */
5065         if (nr_to_scan == 0) {
5066                 spin_lock(&shrink_list_lock);
5067                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5068                         struct drm_device *dev = dev_priv->dev;
5069
5070                         if (mutex_trylock(&dev->struct_mutex)) {
5071                                 list_for_each_entry(obj_priv,
5072                                                     &dev_priv->mm.inactive_list,
5073                                                     list)
5074                                         cnt++;
5075                                 mutex_unlock(&dev->struct_mutex);
5076                         }
5077                 }
5078                 spin_unlock(&shrink_list_lock);
5079
5080                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5081         }
5082
5083         spin_lock(&shrink_list_lock);
5084
5085         /* first scan for clean buffers */
5086         list_for_each_entry_safe(dev_priv, next_dev,
5087                                  &shrink_list, mm.shrink_list) {
5088                 struct drm_device *dev = dev_priv->dev;
5089
5090                 if (! mutex_trylock(&dev->struct_mutex))
5091                         continue;
5092
5093                 spin_unlock(&shrink_list_lock);
5094
5095                 i915_gem_retire_requests(dev);
5096
5097                 list_for_each_entry_safe(obj_priv, next_obj,
5098                                          &dev_priv->mm.inactive_list,
5099                                          list) {
5100                         if (i915_gem_object_is_purgeable(obj_priv)) {
5101                                 i915_gem_object_unbind(obj_priv->obj);
5102                                 if (--nr_to_scan <= 0)
5103                                         break;
5104                         }
5105                 }
5106
5107                 spin_lock(&shrink_list_lock);
5108                 mutex_unlock(&dev->struct_mutex);
5109
5110                 would_deadlock = 0;
5111
5112                 if (nr_to_scan <= 0)
5113                         break;
5114         }
5115
5116         /* second pass, evict/count anything still on the inactive list */
5117         list_for_each_entry_safe(dev_priv, next_dev,
5118                                  &shrink_list, mm.shrink_list) {
5119                 struct drm_device *dev = dev_priv->dev;
5120
5121                 if (! mutex_trylock(&dev->struct_mutex))
5122                         continue;
5123
5124                 spin_unlock(&shrink_list_lock);
5125
5126                 list_for_each_entry_safe(obj_priv, next_obj,
5127                                          &dev_priv->mm.inactive_list,
5128                                          list) {
5129                         if (nr_to_scan > 0) {
5130                                 i915_gem_object_unbind(obj_priv->obj);
5131                                 nr_to_scan--;
5132                         } else
5133                                 cnt++;
5134                 }
5135
5136                 spin_lock(&shrink_list_lock);
5137                 mutex_unlock(&dev->struct_mutex);
5138
5139                 would_deadlock = 0;
5140         }
5141
5142         spin_unlock(&shrink_list_lock);
5143
5144         if (would_deadlock)
5145                 return -1;
5146         else if (cnt > 0)
5147                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5148         else
5149                 return 0;
5150 }
5151
5152 static struct shrinker shrinker = {
5153         .shrink = i915_gem_shrink,
5154         .seeks = DEFAULT_SEEKS,
5155 };
5156
5157 __init void
5158 i915_gem_shrinker_init(void)
5159 {
5160     register_shrinker(&shrinker);
5161 }
5162
5163 __exit void
5164 i915_gem_shrinker_exit(void)
5165 {
5166     unregister_shrinker(&shrinker);
5167 }